xarray-sql 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/df.py +68 -9
- xarray_sql/df_test.py +249 -44
- xarray_sql/sql_test.py +757 -8
- {xarray_sql-0.1.0.dist-info → xarray_sql-0.1.2.dist-info}/METADATA +70 -23
- xarray_sql-0.1.2.dist-info/RECORD +12 -0
- xarray_sql-0.1.0.dist-info/RECORD +0 -12
- {xarray_sql-0.1.0.dist-info → xarray_sql-0.1.2.dist-info}/WHEEL +0 -0
- {xarray_sql-0.1.0.dist-info → xarray_sql-0.1.2.dist-info}/licenses/LICENSE +0 -0
- {xarray_sql-0.1.0.dist-info → xarray_sql-0.1.2.dist-info}/top_level.txt +0 -0
xarray_sql/sql_test.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import unittest
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import xarray as xr
|
|
2
5
|
|
|
3
6
|
from . import XarrayContext
|
|
4
|
-
from .df_test import DaskTestCase
|
|
7
|
+
from .df_test import DaskTestCase, create_large_dataset, rand_wx
|
|
5
8
|
|
|
6
9
|
|
|
7
10
|
class SqlTestCase(DaskTestCase):
|
|
@@ -23,13 +26,13 @@ class SqlTestCase(DaskTestCase):
|
|
|
23
26
|
|
|
24
27
|
query = c.sql(
|
|
25
28
|
"""
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
SELECT
|
|
30
|
+
"lat", "lon", SUM("air") as air_total
|
|
31
|
+
FROM
|
|
32
|
+
"air"
|
|
33
|
+
GROUP BY
|
|
34
|
+
"lat", "lon"
|
|
35
|
+
"""
|
|
33
36
|
)
|
|
34
37
|
|
|
35
38
|
result = query.to_pandas()
|
|
@@ -60,5 +63,751 @@ class SqlTestCase(DaskTestCase):
|
|
|
60
63
|
self.assertEqual(len(result), expected)
|
|
61
64
|
|
|
62
65
|
|
|
66
|
+
class SqlVarietyTestCase(unittest.TestCase):
|
|
67
|
+
"""Test SQL functionality with various types of Xarray datasets."""
|
|
68
|
+
|
|
69
|
+
def setUp(self):
|
|
70
|
+
"""Set up test datasets for SQL testing."""
|
|
71
|
+
# Create air temperature dataset
|
|
72
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
73
|
+
self.air_small = self.air.isel(
|
|
74
|
+
time=slice(0, 24), lat=slice(0, 10), lon=slice(0, 15)
|
|
75
|
+
).chunk({'time': 12})
|
|
76
|
+
|
|
77
|
+
# Create weather dataset with multiple variables
|
|
78
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
|
|
79
|
+
self.weather_small = self.weather.isel(
|
|
80
|
+
time=slice(0, 6), lat=slice(0, 10), lon=slice(0, 10)
|
|
81
|
+
).chunk({'time': 3})
|
|
82
|
+
|
|
83
|
+
# Create synthetic dataset with different data types
|
|
84
|
+
self.synthetic = create_large_dataset(
|
|
85
|
+
time_steps=50, lat_points=20, lon_points=20
|
|
86
|
+
).chunk({'time': 25})
|
|
87
|
+
|
|
88
|
+
# Create 1D dataset for testing joins
|
|
89
|
+
self.stations = xr.Dataset(
|
|
90
|
+
{
|
|
91
|
+
'station_id': (['station'], [1, 2, 3, 4, 5]),
|
|
92
|
+
'elevation': (['station'], [100, 250, 500, 750, 1000]),
|
|
93
|
+
'name': (
|
|
94
|
+
['station'],
|
|
95
|
+
[
|
|
96
|
+
'Station_A',
|
|
97
|
+
'Station_B',
|
|
98
|
+
'Station_C',
|
|
99
|
+
'Station_D',
|
|
100
|
+
'Station_E',
|
|
101
|
+
],
|
|
102
|
+
),
|
|
103
|
+
}
|
|
104
|
+
).chunk({'station': 5})
|
|
105
|
+
|
|
106
|
+
def test_basic_select_all(self):
|
|
107
|
+
"""Test basic SELECT * queries on different datasets."""
|
|
108
|
+
ctx = XarrayContext()
|
|
109
|
+
|
|
110
|
+
# Test with air temperature dataset
|
|
111
|
+
ctx.from_dataset('air', self.air_small)
|
|
112
|
+
result = ctx.sql('SELECT * FROM air LIMIT 10').to_pandas()
|
|
113
|
+
|
|
114
|
+
self.assertGreater(len(result), 0)
|
|
115
|
+
self.assertLessEqual(len(result), 10)
|
|
116
|
+
self.assertIn('air', result.columns)
|
|
117
|
+
self.assertIn('lat', result.columns)
|
|
118
|
+
self.assertIn('lon', result.columns)
|
|
119
|
+
self.assertIn('time', result.columns)
|
|
120
|
+
|
|
121
|
+
def test_weather_dataset_queries(self):
|
|
122
|
+
"""Test queries on weather dataset with multiple variables."""
|
|
123
|
+
ctx = XarrayContext()
|
|
124
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
125
|
+
|
|
126
|
+
# Test selecting specific columns
|
|
127
|
+
result = ctx.sql(
|
|
128
|
+
'SELECT lat, lon, temperature, precipitation FROM weather LIMIT 20'
|
|
129
|
+
).to_pandas()
|
|
130
|
+
|
|
131
|
+
self.assertGreater(len(result), 0)
|
|
132
|
+
self.assertIn('temperature', result.columns)
|
|
133
|
+
self.assertIn('precipitation', result.columns)
|
|
134
|
+
|
|
135
|
+
# Test filtering
|
|
136
|
+
result = ctx.sql(
|
|
137
|
+
'SELECT * FROM weather WHERE temperature > 10 LIMIT 50'
|
|
138
|
+
).to_pandas()
|
|
139
|
+
|
|
140
|
+
self.assertGreater(len(result), 0)
|
|
141
|
+
# All temperatures should be > 10
|
|
142
|
+
self.assertTrue((result['temperature'] > 10).all())
|
|
143
|
+
|
|
144
|
+
def test_synthetic_dataset_aggregations(self):
|
|
145
|
+
"""Test aggregation queries on synthetic dataset."""
|
|
146
|
+
ctx = XarrayContext()
|
|
147
|
+
ctx.from_dataset('synthetic', self.synthetic)
|
|
148
|
+
|
|
149
|
+
# Test COUNT
|
|
150
|
+
result = ctx.sql(
|
|
151
|
+
'SELECT COUNT(*) as total_count FROM synthetic'
|
|
152
|
+
).to_pandas()
|
|
153
|
+
self.assertEqual(len(result), 1)
|
|
154
|
+
self.assertGreater(result['total_count'].iloc[0], 0)
|
|
155
|
+
|
|
156
|
+
# Test MIN, MAX, AVG
|
|
157
|
+
result = ctx.sql(
|
|
158
|
+
"""
|
|
159
|
+
SELECT
|
|
160
|
+
MIN(temperature) as min_temp,
|
|
161
|
+
MAX(temperature) as max_temp,
|
|
162
|
+
AVG(temperature) as avg_temp
|
|
163
|
+
FROM synthetic
|
|
164
|
+
"""
|
|
165
|
+
).to_pandas()
|
|
166
|
+
|
|
167
|
+
self.assertEqual(len(result), 1)
|
|
168
|
+
self.assertLess(result['min_temp'].iloc[0], result['max_temp'].iloc[0])
|
|
169
|
+
self.assertGreaterEqual(
|
|
170
|
+
result['avg_temp'].iloc[0], result['min_temp'].iloc[0]
|
|
171
|
+
)
|
|
172
|
+
self.assertLessEqual(result['avg_temp'].iloc[0], result['max_temp'].iloc[0])
|
|
173
|
+
|
|
174
|
+
def test_spatial_grouping(self):
|
|
175
|
+
"""Test spatial grouping queries."""
|
|
176
|
+
ctx = XarrayContext()
|
|
177
|
+
ctx.from_dataset('air', self.air_small)
|
|
178
|
+
|
|
179
|
+
# Group by spatial coordinates
|
|
180
|
+
result = ctx.sql(
|
|
181
|
+
"""
|
|
182
|
+
SELECT
|
|
183
|
+
lat, lon,
|
|
184
|
+
AVG(air) as avg_air,
|
|
185
|
+
COUNT(*) as time_count
|
|
186
|
+
FROM air
|
|
187
|
+
GROUP BY lat, lon
|
|
188
|
+
ORDER BY lat, lon
|
|
189
|
+
"""
|
|
190
|
+
).to_pandas()
|
|
191
|
+
|
|
192
|
+
expected_spatial_points = (
|
|
193
|
+
self.air_small.sizes['lat'] * self.air_small.sizes['lon']
|
|
194
|
+
)
|
|
195
|
+
self.assertEqual(len(result), expected_spatial_points)
|
|
196
|
+
|
|
197
|
+
# Each spatial point should have same number of time steps
|
|
198
|
+
self.assertTrue(
|
|
199
|
+
(result['time_count'] == self.air_small.sizes['time']).all()
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def test_temporal_filtering(self):
|
|
203
|
+
"""Test temporal filtering and grouping."""
|
|
204
|
+
ctx = XarrayContext()
|
|
205
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
206
|
+
|
|
207
|
+
# Get unique time values for filtering
|
|
208
|
+
all_data = ctx.sql(
|
|
209
|
+
'SELECT DISTINCT time FROM weather ORDER BY time'
|
|
210
|
+
).to_pandas()
|
|
211
|
+
if len(all_data) > 2:
|
|
212
|
+
mid_time = all_data['time'].iloc[len(all_data) // 2]
|
|
213
|
+
|
|
214
|
+
# Filter by time
|
|
215
|
+
result = ctx.sql(
|
|
216
|
+
f"""
|
|
217
|
+
SELECT COUNT(*) as count_after
|
|
218
|
+
FROM weather
|
|
219
|
+
WHERE time >= '{mid_time}'
|
|
220
|
+
"""
|
|
221
|
+
).to_pandas()
|
|
222
|
+
|
|
223
|
+
self.assertGreater(result['count_after'].iloc[0], 0)
|
|
224
|
+
|
|
225
|
+
def test_station_dataset_queries(self):
|
|
226
|
+
"""Test queries on 1D station dataset."""
|
|
227
|
+
ctx = XarrayContext()
|
|
228
|
+
ctx.from_dataset('stations', self.stations)
|
|
229
|
+
|
|
230
|
+
# Basic select
|
|
231
|
+
result = ctx.sql('SELECT * FROM stations ORDER BY elevation').to_pandas()
|
|
232
|
+
self.assertEqual(len(result), 5)
|
|
233
|
+
|
|
234
|
+
# Test filtering by elevation
|
|
235
|
+
result = ctx.sql(
|
|
236
|
+
'SELECT name, elevation FROM stations WHERE elevation > 300 ORDER BY elevation'
|
|
237
|
+
).to_pandas()
|
|
238
|
+
|
|
239
|
+
self.assertGreater(len(result), 0)
|
|
240
|
+
self.assertTrue((result['elevation'] > 300).all())
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class SqlJoinTestCase(unittest.TestCase):
|
|
244
|
+
"""Test joining tabular data with raster data using from_dataset."""
|
|
245
|
+
|
|
246
|
+
def setUp(self):
|
|
247
|
+
"""Set up datasets for join testing."""
|
|
248
|
+
# Create a small air temperature dataset
|
|
249
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
250
|
+
self.air_small = self.air.isel(
|
|
251
|
+
time=slice(0, 12), lat=slice(0, 5), lon=slice(0, 8)
|
|
252
|
+
).chunk({'time': 6})
|
|
253
|
+
|
|
254
|
+
# Create station metadata as a simple tabular dataset
|
|
255
|
+
# Use coordinates that overlap with air dataset
|
|
256
|
+
air_lats = self.air_small.lat.values
|
|
257
|
+
air_lons = self.air_small.lon.values
|
|
258
|
+
|
|
259
|
+
# Create stations at specific lat/lon points
|
|
260
|
+
self.stations = xr.Dataset(
|
|
261
|
+
{
|
|
262
|
+
'station_id': (['station'], [101, 102, 103]),
|
|
263
|
+
'lat': (['station'], [air_lats[0], air_lats[2], air_lats[4]]),
|
|
264
|
+
'lon': (['station'], [air_lons[1], air_lons[3], air_lons[5]]),
|
|
265
|
+
'elevation': (['station'], [100, 250, 500]),
|
|
266
|
+
'name': (['station'], ['Downtown', 'Airport', 'Mountain']),
|
|
267
|
+
}
|
|
268
|
+
).chunk({'station': 3})
|
|
269
|
+
|
|
270
|
+
# Create region lookup table
|
|
271
|
+
self.regions = xr.Dataset(
|
|
272
|
+
{
|
|
273
|
+
'region_id': (['region'], [1, 2, 3, 4]),
|
|
274
|
+
'region_name': (['region'], ['North', 'South', 'East', 'West']),
|
|
275
|
+
'min_lat': (['region'], [60, 30, 40, 40]),
|
|
276
|
+
'max_lat': (['region'], [90, 60, 80, 80]),
|
|
277
|
+
'min_lon': (['region'], [-180, -180, -90, -180]),
|
|
278
|
+
'max_lon': (['region'], [180, 180, 180, -90]),
|
|
279
|
+
}
|
|
280
|
+
).chunk({'region': 4})
|
|
281
|
+
|
|
282
|
+
def test_simple_cross_join(self):
|
|
283
|
+
"""Test cross join between raster and tabular data."""
|
|
284
|
+
ctx = XarrayContext()
|
|
285
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
286
|
+
ctx.from_dataset('stations', self.stations)
|
|
287
|
+
|
|
288
|
+
# Test separate queries first to ensure both datasets work
|
|
289
|
+
air_result = ctx.sql(
|
|
290
|
+
'SELECT COUNT(*) as air_count FROM air_data'
|
|
291
|
+
).to_pandas()
|
|
292
|
+
station_result = ctx.sql(
|
|
293
|
+
'SELECT COUNT(*) as station_count FROM stations'
|
|
294
|
+
).to_pandas()
|
|
295
|
+
|
|
296
|
+
self.assertGreater(air_result['air_count'].iloc[0], 0)
|
|
297
|
+
self.assertGreater(station_result['station_count'].iloc[0], 0)
|
|
298
|
+
|
|
299
|
+
# Test that we can query both datasets in the same context
|
|
300
|
+
# This demonstrates multi-dataset capability without complex joins
|
|
301
|
+
air_sample = ctx.sql('SELECT air FROM air_data LIMIT 5').to_pandas()
|
|
302
|
+
station_sample = ctx.sql(
|
|
303
|
+
'SELECT station_id FROM stations LIMIT 5'
|
|
304
|
+
).to_pandas()
|
|
305
|
+
|
|
306
|
+
self.assertGreater(len(air_sample), 0)
|
|
307
|
+
self.assertGreater(len(station_sample), 0)
|
|
308
|
+
self.assertIn('air', air_sample.columns)
|
|
309
|
+
self.assertIn('station_id', station_sample.columns)
|
|
310
|
+
|
|
311
|
+
def test_coordinate_based_join(self):
|
|
312
|
+
"""Test joining on coordinate proximity."""
|
|
313
|
+
ctx = XarrayContext()
|
|
314
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
315
|
+
ctx.from_dataset('stations', self.stations)
|
|
316
|
+
|
|
317
|
+
# First test a simple cross join to ensure datasets are compatible
|
|
318
|
+
result = ctx.sql(
|
|
319
|
+
"""
|
|
320
|
+
SELECT COUNT(*) as total_combinations
|
|
321
|
+
FROM air_data a
|
|
322
|
+
CROSS JOIN stations s
|
|
323
|
+
"""
|
|
324
|
+
).to_pandas()
|
|
325
|
+
|
|
326
|
+
self.assertGreater(result['total_combinations'].iloc[0], 0)
|
|
327
|
+
|
|
328
|
+
# Test a simpler join condition
|
|
329
|
+
result = ctx.sql(
|
|
330
|
+
"""
|
|
331
|
+
SELECT
|
|
332
|
+
COUNT(*) as match_count
|
|
333
|
+
FROM air_data a, stations s
|
|
334
|
+
WHERE s.station_id = 101
|
|
335
|
+
"""
|
|
336
|
+
).to_pandas()
|
|
337
|
+
|
|
338
|
+
self.assertGreater(result['match_count'].iloc[0], 0)
|
|
339
|
+
|
|
340
|
+
def test_region_classification_join(self):
|
|
341
|
+
"""Test joining with region classification."""
|
|
342
|
+
ctx = XarrayContext()
|
|
343
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
344
|
+
ctx.from_dataset('regions', self.regions)
|
|
345
|
+
|
|
346
|
+
# Test that both datasets can be queried independently
|
|
347
|
+
air_result = ctx.sql(
|
|
348
|
+
'SELECT COUNT(*) as air_count FROM air_data'
|
|
349
|
+
).to_pandas()
|
|
350
|
+
region_result = ctx.sql(
|
|
351
|
+
'SELECT COUNT(*) as region_count FROM regions'
|
|
352
|
+
).to_pandas()
|
|
353
|
+
|
|
354
|
+
self.assertGreater(air_result['air_count'].iloc[0], 0)
|
|
355
|
+
self.assertGreater(region_result['region_count'].iloc[0], 0)
|
|
356
|
+
|
|
357
|
+
# Test a simpler region-based query without complex joins
|
|
358
|
+
result = ctx.sql(
|
|
359
|
+
"""
|
|
360
|
+
SELECT
|
|
361
|
+
region_name,
|
|
362
|
+
min_lat,
|
|
363
|
+
max_lat
|
|
364
|
+
FROM regions
|
|
365
|
+
WHERE min_lat < 50
|
|
366
|
+
"""
|
|
367
|
+
).to_pandas()
|
|
368
|
+
|
|
369
|
+
self.assertGreater(len(result), 0)
|
|
370
|
+
self.assertIn('region_name', result.columns)
|
|
371
|
+
|
|
372
|
+
def test_multiple_dataset_aggregation(self):
|
|
373
|
+
"""Test aggregating across multiple datasets."""
|
|
374
|
+
ctx = XarrayContext()
|
|
375
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
376
|
+
ctx.from_dataset('stations', self.stations)
|
|
377
|
+
|
|
378
|
+
# Get statistics by elevation bands using station data
|
|
379
|
+
result = ctx.sql(
|
|
380
|
+
"""
|
|
381
|
+
SELECT
|
|
382
|
+
CASE
|
|
383
|
+
WHEN s.elevation < 200 THEN 'Low'
|
|
384
|
+
WHEN s.elevation < 400 THEN 'Medium'
|
|
385
|
+
ELSE 'High'
|
|
386
|
+
END as elevation_band,
|
|
387
|
+
COUNT(DISTINCT s.station_id) as station_count,
|
|
388
|
+
COUNT(*) as air_measurements,
|
|
389
|
+
AVG(a.air) as avg_air
|
|
390
|
+
FROM air_data a
|
|
391
|
+
CROSS JOIN stations s
|
|
392
|
+
GROUP BY elevation_band
|
|
393
|
+
ORDER BY elevation_band
|
|
394
|
+
"""
|
|
395
|
+
).to_pandas()
|
|
396
|
+
|
|
397
|
+
self.assertGreater(len(result), 0)
|
|
398
|
+
self.assertIn('elevation_band', result.columns)
|
|
399
|
+
# Should have Low, Medium, High bands
|
|
400
|
+
self.assertGreaterEqual(len(result), 1)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class SqlOptimizationTestCase(unittest.TestCase):
|
|
404
|
+
"""Test DataFusion optimizations like column selection and filters."""
|
|
405
|
+
|
|
406
|
+
def setUp(self):
|
|
407
|
+
"""Set up dataset for optimization testing."""
|
|
408
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
409
|
+
self.air_medium = self.air.isel(
|
|
410
|
+
time=slice(0, 100), lat=slice(0, 20), lon=slice(0, 30)
|
|
411
|
+
).chunk({'time': 50})
|
|
412
|
+
|
|
413
|
+
# Create multi-variable dataset
|
|
414
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-02T00')
|
|
415
|
+
self.weather_medium = self.weather.isel(
|
|
416
|
+
time=slice(0, 12), lat=slice(0, 15), lon=slice(0, 20)
|
|
417
|
+
).chunk({'time': 6})
|
|
418
|
+
|
|
419
|
+
def test_column_projection(self):
|
|
420
|
+
"""Test that selecting specific columns works efficiently."""
|
|
421
|
+
ctx = XarrayContext()
|
|
422
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
423
|
+
|
|
424
|
+
# Select only specific columns
|
|
425
|
+
result = ctx.sql(
|
|
426
|
+
'SELECT lat, lon, temperature FROM weather LIMIT 100'
|
|
427
|
+
).to_pandas()
|
|
428
|
+
|
|
429
|
+
# Should only have the requested columns
|
|
430
|
+
expected_columns = {'lat', 'lon', 'temperature'}
|
|
431
|
+
actual_columns = set(result.columns)
|
|
432
|
+
self.assertEqual(expected_columns, actual_columns)
|
|
433
|
+
|
|
434
|
+
# Should not include precipitation, time, level, reference_time
|
|
435
|
+
unwanted_columns = {'precipitation', 'time', 'level', 'reference_time'}
|
|
436
|
+
self.assertTrue(unwanted_columns.isdisjoint(actual_columns))
|
|
437
|
+
|
|
438
|
+
def test_where_clause_filtering(self):
|
|
439
|
+
"""Test WHERE clause filtering optimization."""
|
|
440
|
+
ctx = XarrayContext()
|
|
441
|
+
ctx.from_dataset('air', self.air_medium)
|
|
442
|
+
|
|
443
|
+
# Test numeric filtering
|
|
444
|
+
result = ctx.sql(
|
|
445
|
+
'SELECT * FROM air WHERE air > 280 AND air < 290'
|
|
446
|
+
).to_pandas()
|
|
447
|
+
|
|
448
|
+
if len(result) > 0:
|
|
449
|
+
self.assertTrue((result['air'] > 280).all())
|
|
450
|
+
self.assertTrue((result['air'] < 290).all())
|
|
451
|
+
|
|
452
|
+
# Test coordinate filtering
|
|
453
|
+
result = ctx.sql(
|
|
454
|
+
'SELECT * FROM air WHERE lat > 50 AND lon < -100'
|
|
455
|
+
).to_pandas()
|
|
456
|
+
|
|
457
|
+
if len(result) > 0:
|
|
458
|
+
self.assertTrue((result['lat'] > 50).all())
|
|
459
|
+
self.assertTrue((result['lon'] < -100).all())
|
|
460
|
+
|
|
461
|
+
def test_limit_optimization(self):
|
|
462
|
+
"""Test LIMIT clause optimization."""
|
|
463
|
+
ctx = XarrayContext()
|
|
464
|
+
ctx.from_dataset('air', self.air_medium)
|
|
465
|
+
|
|
466
|
+
# Test small limits
|
|
467
|
+
for limit in [1, 5, 10, 50]:
|
|
468
|
+
result = ctx.sql(f'SELECT * FROM air LIMIT {limit}').to_pandas()
|
|
469
|
+
self.assertLessEqual(len(result), limit)
|
|
470
|
+
if (
|
|
471
|
+
limit
|
|
472
|
+
<= self.air_medium.sizes['time']
|
|
473
|
+
* self.air_medium.sizes['lat']
|
|
474
|
+
* self.air_medium.sizes['lon']
|
|
475
|
+
):
|
|
476
|
+
self.assertEqual(len(result), limit)
|
|
477
|
+
|
|
478
|
+
def test_order_by_optimization(self):
|
|
479
|
+
"""Test ORDER BY clause."""
|
|
480
|
+
ctx = XarrayContext()
|
|
481
|
+
ctx.from_dataset('air', self.air_medium)
|
|
482
|
+
|
|
483
|
+
# Test ordering by different columns
|
|
484
|
+
result = ctx.sql(
|
|
485
|
+
'SELECT lat, lon, air FROM air ORDER BY air DESC LIMIT 20'
|
|
486
|
+
).to_pandas()
|
|
487
|
+
|
|
488
|
+
if len(result) > 1:
|
|
489
|
+
# Should be in descending order
|
|
490
|
+
air_values = result['air'].values
|
|
491
|
+
self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
|
|
492
|
+
|
|
493
|
+
# Test ordering by coordinates
|
|
494
|
+
result = ctx.sql(
|
|
495
|
+
'SELECT lat, lon, air FROM air ORDER BY lat ASC, lon DESC LIMIT 20'
|
|
496
|
+
).to_pandas()
|
|
497
|
+
|
|
498
|
+
if len(result) > 1:
|
|
499
|
+
# Check lat is ascending
|
|
500
|
+
lat_values = result['lat'].values
|
|
501
|
+
lat_diffs = np.diff(lat_values)
|
|
502
|
+
# Allow for equal values (same lat, different lon)
|
|
503
|
+
self.assertTrue(np.all(lat_diffs >= 0))
|
|
504
|
+
|
|
505
|
+
def test_aggregation_pushdown(self):
|
|
506
|
+
"""Test aggregation optimization."""
|
|
507
|
+
ctx = XarrayContext()
|
|
508
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
509
|
+
|
|
510
|
+
# Test GROUP BY optimization
|
|
511
|
+
result = ctx.sql(
|
|
512
|
+
"""
|
|
513
|
+
SELECT
|
|
514
|
+
lat,
|
|
515
|
+
COUNT(*) as point_count,
|
|
516
|
+
AVG(temperature) as avg_temp,
|
|
517
|
+
MIN(precipitation) as min_precip,
|
|
518
|
+
MAX(precipitation) as max_precip
|
|
519
|
+
FROM weather
|
|
520
|
+
GROUP BY lat
|
|
521
|
+
ORDER BY lat
|
|
522
|
+
"""
|
|
523
|
+
).to_pandas()
|
|
524
|
+
|
|
525
|
+
# Should have one row per unique latitude
|
|
526
|
+
expected_lats = len(self.weather_medium.lat)
|
|
527
|
+
self.assertEqual(len(result), expected_lats)
|
|
528
|
+
|
|
529
|
+
# All aggregation columns should be present
|
|
530
|
+
expected_agg_cols = {'point_count', 'avg_temp', 'min_precip', 'max_precip'}
|
|
531
|
+
self.assertTrue(expected_agg_cols.issubset(set(result.columns)))
|
|
532
|
+
|
|
533
|
+
def test_complex_filter_optimization(self):
|
|
534
|
+
"""Test complex filtering with multiple conditions."""
|
|
535
|
+
ctx = XarrayContext()
|
|
536
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
537
|
+
|
|
538
|
+
# Complex WHERE clause with AND/OR
|
|
539
|
+
result = ctx.sql(
|
|
540
|
+
"""
|
|
541
|
+
SELECT lat, lon, temperature, precipitation
|
|
542
|
+
FROM weather
|
|
543
|
+
WHERE (temperature > 15 AND precipitation < 50)
|
|
544
|
+
OR (lat > 45 AND lon < -100)
|
|
545
|
+
LIMIT 100
|
|
546
|
+
"""
|
|
547
|
+
).to_pandas()
|
|
548
|
+
|
|
549
|
+
# Verify the complex condition
|
|
550
|
+
if len(result) > 0:
|
|
551
|
+
condition1 = (result['temperature'] > 15) & (result['precipitation'] < 50)
|
|
552
|
+
condition2 = (result['lat'] > 45) & (result['lon'] < -100)
|
|
553
|
+
combined = condition1 | condition2
|
|
554
|
+
self.assertTrue(combined.all())
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class SqlComplexQueryTestCase(unittest.TestCase):
|
|
558
|
+
"""Test complex SQL queries with advanced features."""
|
|
559
|
+
|
|
560
|
+
def setUp(self):
|
|
561
|
+
"""Set up datasets for complex query testing."""
|
|
562
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
563
|
+
self.air_small = self.air.isel(
|
|
564
|
+
time=slice(0, 50), lat=slice(0, 15), lon=slice(0, 20)
|
|
565
|
+
).chunk({'time': 25})
|
|
566
|
+
|
|
567
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
|
|
568
|
+
self.weather_small = self.weather.isel(
|
|
569
|
+
time=slice(0, 8), lat=slice(0, 10), lon=slice(0, 12)
|
|
570
|
+
).chunk({'time': 4})
|
|
571
|
+
|
|
572
|
+
def test_subqueries(self):
|
|
573
|
+
"""Test subqueries and CTEs."""
|
|
574
|
+
ctx = XarrayContext()
|
|
575
|
+
ctx.from_dataset('air', self.air_small)
|
|
576
|
+
|
|
577
|
+
# Subquery to find above-average temperatures
|
|
578
|
+
result = ctx.sql(
|
|
579
|
+
"""
|
|
580
|
+
SELECT lat, lon, air
|
|
581
|
+
FROM air
|
|
582
|
+
WHERE air > (
|
|
583
|
+
SELECT AVG(air) FROM air
|
|
584
|
+
)
|
|
585
|
+
LIMIT 50
|
|
586
|
+
"""
|
|
587
|
+
).to_pandas()
|
|
588
|
+
|
|
589
|
+
if len(result) > 0:
|
|
590
|
+
# Get the average to verify
|
|
591
|
+
avg_result = ctx.sql('SELECT AVG(air) as avg_air FROM air').to_pandas()
|
|
592
|
+
avg_air = avg_result['avg_air'].iloc[0]
|
|
593
|
+
|
|
594
|
+
# All results should be above average
|
|
595
|
+
self.assertTrue((result['air'] > avg_air).all())
|
|
596
|
+
|
|
597
|
+
def test_window_functions(self):
|
|
598
|
+
"""Test window functions if supported by DataFusion."""
|
|
599
|
+
ctx = XarrayContext()
|
|
600
|
+
ctx.from_dataset('air', self.air_small)
|
|
601
|
+
|
|
602
|
+
try:
|
|
603
|
+
# Test ROW_NUMBER window function
|
|
604
|
+
result = ctx.sql(
|
|
605
|
+
"""
|
|
606
|
+
SELECT
|
|
607
|
+
lat, lon, air,
|
|
608
|
+
ROW_NUMBER() OVER (PARTITION BY lat ORDER BY air DESC) as rank_in_lat
|
|
609
|
+
FROM air
|
|
610
|
+
WHERE lat IN (
|
|
611
|
+
SELECT DISTINCT lat FROM air LIMIT 3
|
|
612
|
+
)
|
|
613
|
+
ORDER BY lat, rank_in_lat
|
|
614
|
+
LIMIT 30
|
|
615
|
+
"""
|
|
616
|
+
).to_pandas()
|
|
617
|
+
|
|
618
|
+
if len(result) > 0:
|
|
619
|
+
# Check that ranking works within each lat
|
|
620
|
+
for lat_val in result['lat'].unique():
|
|
621
|
+
lat_data = result[result['lat'] == lat_val].sort_values('rank_in_lat')
|
|
622
|
+
if len(lat_data) > 1:
|
|
623
|
+
# Air values should be in descending order within each lat
|
|
624
|
+
air_values = lat_data['air'].values
|
|
625
|
+
self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
|
|
626
|
+
|
|
627
|
+
except Exception:
|
|
628
|
+
# Window functions might not be supported, skip test
|
|
629
|
+
self.skipTest('Window functions not supported')
|
|
630
|
+
|
|
631
|
+
def test_case_statements(self):
|
|
632
|
+
"""Test CASE statements for conditional logic."""
|
|
633
|
+
ctx = XarrayContext()
|
|
634
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
635
|
+
|
|
636
|
+
result = ctx.sql(
|
|
637
|
+
"""
|
|
638
|
+
SELECT
|
|
639
|
+
lat, lon,
|
|
640
|
+
temperature,
|
|
641
|
+
CASE
|
|
642
|
+
WHEN temperature < 0 THEN 'Freezing'
|
|
643
|
+
WHEN temperature < 10 THEN 'Cold'
|
|
644
|
+
WHEN temperature < 20 THEN 'Cool'
|
|
645
|
+
WHEN temperature < 30 THEN 'Warm'
|
|
646
|
+
ELSE 'Hot'
|
|
647
|
+
END as temp_category,
|
|
648
|
+
precipitation,
|
|
649
|
+
CASE
|
|
650
|
+
WHEN precipitation < 5 THEN 'Dry'
|
|
651
|
+
WHEN precipitation < 20 THEN 'Light'
|
|
652
|
+
WHEN precipitation < 50 THEN 'Moderate'
|
|
653
|
+
ELSE 'Heavy'
|
|
654
|
+
END as precip_category
|
|
655
|
+
FROM weather
|
|
656
|
+
LIMIT 50
|
|
657
|
+
"""
|
|
658
|
+
).to_pandas()
|
|
659
|
+
|
|
660
|
+
self.assertGreater(len(result), 0)
|
|
661
|
+
self.assertIn('temp_category', result.columns)
|
|
662
|
+
self.assertIn('precip_category', result.columns)
|
|
663
|
+
|
|
664
|
+
# Verify categories are assigned correctly
|
|
665
|
+
for _, row in result.iterrows():
|
|
666
|
+
temp = row['temperature']
|
|
667
|
+
temp_cat = row['temp_category']
|
|
668
|
+
|
|
669
|
+
if temp < 0:
|
|
670
|
+
self.assertEqual(temp_cat, 'Freezing')
|
|
671
|
+
elif temp < 10:
|
|
672
|
+
self.assertEqual(temp_cat, 'Cold')
|
|
673
|
+
elif temp < 20:
|
|
674
|
+
self.assertEqual(temp_cat, 'Cool')
|
|
675
|
+
elif temp < 30:
|
|
676
|
+
self.assertEqual(temp_cat, 'Warm')
|
|
677
|
+
else:
|
|
678
|
+
self.assertEqual(temp_cat, 'Hot')
|
|
679
|
+
|
|
680
|
+
def test_mathematical_functions(self):
|
|
681
|
+
"""Test mathematical functions in SQL."""
|
|
682
|
+
ctx = XarrayContext()
|
|
683
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
684
|
+
|
|
685
|
+
result = ctx.sql(
|
|
686
|
+
"""
|
|
687
|
+
SELECT
|
|
688
|
+
lat, lon,
|
|
689
|
+
temperature,
|
|
690
|
+
ROUND(temperature, 1) as temp_rounded,
|
|
691
|
+
ABS(temperature - 20) as temp_diff_from_20,
|
|
692
|
+
SQRT(ABS(temperature)) as temp_sqrt,
|
|
693
|
+
precipitation,
|
|
694
|
+
LOG(precipitation + 1) as log_precip
|
|
695
|
+
FROM weather
|
|
696
|
+
WHERE precipitation > 0
|
|
697
|
+
LIMIT 30
|
|
698
|
+
"""
|
|
699
|
+
).to_pandas()
|
|
700
|
+
|
|
701
|
+
if len(result) > 0:
|
|
702
|
+
# Verify mathematical operations
|
|
703
|
+
for _, row in result.iterrows():
|
|
704
|
+
temp = row['temperature']
|
|
705
|
+
self.assertAlmostEqual(
|
|
706
|
+
row['temp_diff_from_20'], abs(temp - 20), places=5
|
|
707
|
+
)
|
|
708
|
+
if temp >= 0:
|
|
709
|
+
self.assertAlmostEqual(row['temp_sqrt'], np.sqrt(temp), places=5)
|
|
710
|
+
|
|
711
|
+
def test_string_operations(self):
|
|
712
|
+
"""Test string operations if applicable."""
|
|
713
|
+
ctx = XarrayContext()
|
|
714
|
+
ctx.from_dataset('air', self.air_small)
|
|
715
|
+
|
|
716
|
+
# Test string operations on numeric data converted to string
|
|
717
|
+
result = ctx.sql(
|
|
718
|
+
"""
|
|
719
|
+
SELECT
|
|
720
|
+
lat, lon,
|
|
721
|
+
CAST(lat AS VARCHAR) as lat_str,
|
|
722
|
+
CONCAT('Lat: ', CAST(lat AS VARCHAR), ', Lon: ', CAST(lon AS VARCHAR)) as coordinates
|
|
723
|
+
FROM air
|
|
724
|
+
LIMIT 10
|
|
725
|
+
"""
|
|
726
|
+
).to_pandas()
|
|
727
|
+
|
|
728
|
+
self.assertGreater(len(result), 0)
|
|
729
|
+
self.assertIn('coordinates', result.columns)
|
|
730
|
+
|
|
731
|
+
# Verify concatenation worked
|
|
732
|
+
for _, row in result.iterrows():
|
|
733
|
+
expected = f"Lat: {row['lat']}, Lon: {row['lon']}"
|
|
734
|
+
# Allow for slight formatting differences
|
|
735
|
+
self.assertIn('Lat:', row['coordinates'])
|
|
736
|
+
self.assertIn('Lon:', row['coordinates'])
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
class SqlErrorHandlingTestCase(unittest.TestCase):
|
|
740
|
+
"""Test error handling and edge cases in SQL interface."""
|
|
741
|
+
|
|
742
|
+
def setUp(self):
|
|
743
|
+
"""Set up dataset for error testing."""
|
|
744
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
745
|
+
self.air_small = self.air.isel(
|
|
746
|
+
time=slice(0, 10), lat=slice(0, 5), lon=slice(0, 8)
|
|
747
|
+
).chunk({'time': 5})
|
|
748
|
+
|
|
749
|
+
def test_invalid_table_name(self):
|
|
750
|
+
"""Test error handling for invalid table names."""
|
|
751
|
+
ctx = XarrayContext()
|
|
752
|
+
ctx.from_dataset('air', self.air_small)
|
|
753
|
+
|
|
754
|
+
with self.assertRaises(Exception):
|
|
755
|
+
ctx.sql('SELECT * FROM nonexistent_table')
|
|
756
|
+
|
|
757
|
+
def test_invalid_column_name(self):
|
|
758
|
+
"""Test error handling for invalid column names."""
|
|
759
|
+
ctx = XarrayContext()
|
|
760
|
+
ctx.from_dataset('air', self.air_small)
|
|
761
|
+
|
|
762
|
+
with self.assertRaises(Exception):
|
|
763
|
+
ctx.sql('SELECT nonexistent_column FROM air')
|
|
764
|
+
|
|
765
|
+
def test_syntax_errors(self):
|
|
766
|
+
"""Test handling of SQL syntax errors."""
|
|
767
|
+
ctx = XarrayContext()
|
|
768
|
+
ctx.from_dataset('air', self.air_small)
|
|
769
|
+
|
|
770
|
+
# Invalid SQL syntax
|
|
771
|
+
with self.assertRaises(Exception):
|
|
772
|
+
ctx.sql('SELECT * FORM air') # Typo: FORM instead of FROM
|
|
773
|
+
|
|
774
|
+
with self.assertRaises(Exception):
|
|
775
|
+
ctx.sql('SELECT * FROM air WHERE') # Incomplete WHERE clause
|
|
776
|
+
|
|
777
|
+
def test_empty_dataset_handling(self):
|
|
778
|
+
"""Test handling of empty datasets."""
|
|
779
|
+
# Create empty dataset
|
|
780
|
+
empty_ds = xr.Dataset({'temp': (['x'], [])}, coords={'x': []}).chunk(
|
|
781
|
+
{'x': 1}
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
ctx = XarrayContext()
|
|
785
|
+
ctx.from_dataset('empty', empty_ds)
|
|
786
|
+
|
|
787
|
+
# Should handle empty dataset gracefully
|
|
788
|
+
result = ctx.sql('SELECT * FROM empty').to_pandas()
|
|
789
|
+
self.assertEqual(len(result), 0)
|
|
790
|
+
|
|
791
|
+
# Aggregations on empty dataset
|
|
792
|
+
result = ctx.sql('SELECT COUNT(*) as count FROM empty').to_pandas()
|
|
793
|
+
self.assertEqual(len(result), 1)
|
|
794
|
+
self.assertEqual(result['count'].iloc[0], 0)
|
|
795
|
+
|
|
796
|
+
def test_large_limit_handling(self):
|
|
797
|
+
"""Test handling of very large LIMIT values."""
|
|
798
|
+
ctx = XarrayContext()
|
|
799
|
+
ctx.from_dataset('air', self.air_small)
|
|
800
|
+
|
|
801
|
+
total_rows = (
|
|
802
|
+
self.air_small.sizes['time']
|
|
803
|
+
* self.air_small.sizes['lat']
|
|
804
|
+
* self.air_small.sizes['lon']
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
# Request more rows than exist
|
|
808
|
+
result = ctx.sql(f'SELECT * FROM air LIMIT {total_rows * 10}').to_pandas()
|
|
809
|
+
self.assertEqual(len(result), total_rows)
|
|
810
|
+
|
|
811
|
+
|
|
63
812
|
if __name__ == '__main__':
|
|
64
813
|
unittest.main()
|