xarray-sql 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/df.py +24 -10
- xarray_sql/sql_test.py +750 -1
- {xarray_sql-0.1.1.dist-info → xarray_sql-0.1.2.dist-info}/METADATA +70 -23
- {xarray_sql-0.1.1.dist-info → xarray_sql-0.1.2.dist-info}/RECORD +7 -7
- {xarray_sql-0.1.1.dist-info → xarray_sql-0.1.2.dist-info}/WHEEL +0 -0
- {xarray_sql-0.1.1.dist-info → xarray_sql-0.1.2.dist-info}/licenses/LICENSE +0 -0
- {xarray_sql-0.1.1.dist-info → xarray_sql-0.1.2.dist-info}/top_level.txt +0 -0
xarray_sql/df.py
CHANGED
|
@@ -150,7 +150,24 @@ def pivot(ds: xr.Dataset) -> pd.DataFrame:
|
|
|
150
150
|
return ds.to_dataframe().reset_index()
|
|
151
151
|
|
|
152
152
|
|
|
153
|
-
def
|
|
153
|
+
def _parse_schema(ds) -> pa.Schema:
|
|
154
|
+
"""Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
|
|
155
|
+
columns = []
|
|
156
|
+
|
|
157
|
+
for coord_name, coord_var in ds.coords.items():
|
|
158
|
+
# Only include dimension coordinates
|
|
159
|
+
if coord_name in ds.dims:
|
|
160
|
+
pa_type = pa.from_numpy_dtype(coord_var.dtype)
|
|
161
|
+
columns.append(pa.field(coord_name, pa_type))
|
|
162
|
+
|
|
163
|
+
for var_name, var in ds.data_vars.items():
|
|
164
|
+
pa_type = pa.from_numpy_dtype(var.dtype)
|
|
165
|
+
columns.append(pa.field(var_name, pa_type))
|
|
166
|
+
|
|
167
|
+
return pa.schema(columns)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
|
|
154
171
|
"""Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
|
|
155
172
|
|
|
156
173
|
Args:
|
|
@@ -162,18 +179,15 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
|
|
|
162
179
|
Returns:
|
|
163
180
|
A PyArrow Table, which is a table representation of the input Dataset.
|
|
164
181
|
"""
|
|
165
|
-
fst = next(iter(ds.values())).dims
|
|
166
|
-
assert all(
|
|
167
|
-
da.dims == fst for da in ds.values()
|
|
168
|
-
), "All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
169
|
-
|
|
170
|
-
blocks = list(block_slices(ds, chunks))
|
|
171
182
|
|
|
172
183
|
def pivot_block(b: Block):
|
|
173
184
|
return pivot(ds.isel(b))
|
|
174
185
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
186
|
+
fst = next(iter(ds.values())).dims
|
|
187
|
+
assert all(
|
|
188
|
+
da.dims == fst for da in ds.values()
|
|
189
|
+
), "All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
178
190
|
|
|
191
|
+
schema = _parse_schema(ds)
|
|
192
|
+
blocks = block_slices(ds, chunks)
|
|
179
193
|
return from_map_batched(pivot_block, blocks, schema=schema)
|
xarray_sql/sql_test.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import unittest
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import xarray as xr
|
|
2
5
|
|
|
3
6
|
from . import XarrayContext
|
|
4
|
-
from .df_test import DaskTestCase
|
|
7
|
+
from .df_test import DaskTestCase, create_large_dataset, rand_wx
|
|
5
8
|
|
|
6
9
|
|
|
7
10
|
class SqlTestCase(DaskTestCase):
|
|
@@ -60,5 +63,751 @@ class SqlTestCase(DaskTestCase):
|
|
|
60
63
|
self.assertEqual(len(result), expected)
|
|
61
64
|
|
|
62
65
|
|
|
66
|
+
class SqlVarietyTestCase(unittest.TestCase):
|
|
67
|
+
"""Test SQL functionality with various types of Xarray datasets."""
|
|
68
|
+
|
|
69
|
+
def setUp(self):
|
|
70
|
+
"""Set up test datasets for SQL testing."""
|
|
71
|
+
# Create air temperature dataset
|
|
72
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
73
|
+
self.air_small = self.air.isel(
|
|
74
|
+
time=slice(0, 24), lat=slice(0, 10), lon=slice(0, 15)
|
|
75
|
+
).chunk({'time': 12})
|
|
76
|
+
|
|
77
|
+
# Create weather dataset with multiple variables
|
|
78
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
|
|
79
|
+
self.weather_small = self.weather.isel(
|
|
80
|
+
time=slice(0, 6), lat=slice(0, 10), lon=slice(0, 10)
|
|
81
|
+
).chunk({'time': 3})
|
|
82
|
+
|
|
83
|
+
# Create synthetic dataset with different data types
|
|
84
|
+
self.synthetic = create_large_dataset(
|
|
85
|
+
time_steps=50, lat_points=20, lon_points=20
|
|
86
|
+
).chunk({'time': 25})
|
|
87
|
+
|
|
88
|
+
# Create 1D dataset for testing joins
|
|
89
|
+
self.stations = xr.Dataset(
|
|
90
|
+
{
|
|
91
|
+
'station_id': (['station'], [1, 2, 3, 4, 5]),
|
|
92
|
+
'elevation': (['station'], [100, 250, 500, 750, 1000]),
|
|
93
|
+
'name': (
|
|
94
|
+
['station'],
|
|
95
|
+
[
|
|
96
|
+
'Station_A',
|
|
97
|
+
'Station_B',
|
|
98
|
+
'Station_C',
|
|
99
|
+
'Station_D',
|
|
100
|
+
'Station_E',
|
|
101
|
+
],
|
|
102
|
+
),
|
|
103
|
+
}
|
|
104
|
+
).chunk({'station': 5})
|
|
105
|
+
|
|
106
|
+
def test_basic_select_all(self):
|
|
107
|
+
"""Test basic SELECT * queries on different datasets."""
|
|
108
|
+
ctx = XarrayContext()
|
|
109
|
+
|
|
110
|
+
# Test with air temperature dataset
|
|
111
|
+
ctx.from_dataset('air', self.air_small)
|
|
112
|
+
result = ctx.sql('SELECT * FROM air LIMIT 10').to_pandas()
|
|
113
|
+
|
|
114
|
+
self.assertGreater(len(result), 0)
|
|
115
|
+
self.assertLessEqual(len(result), 10)
|
|
116
|
+
self.assertIn('air', result.columns)
|
|
117
|
+
self.assertIn('lat', result.columns)
|
|
118
|
+
self.assertIn('lon', result.columns)
|
|
119
|
+
self.assertIn('time', result.columns)
|
|
120
|
+
|
|
121
|
+
def test_weather_dataset_queries(self):
|
|
122
|
+
"""Test queries on weather dataset with multiple variables."""
|
|
123
|
+
ctx = XarrayContext()
|
|
124
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
125
|
+
|
|
126
|
+
# Test selecting specific columns
|
|
127
|
+
result = ctx.sql(
|
|
128
|
+
'SELECT lat, lon, temperature, precipitation FROM weather LIMIT 20'
|
|
129
|
+
).to_pandas()
|
|
130
|
+
|
|
131
|
+
self.assertGreater(len(result), 0)
|
|
132
|
+
self.assertIn('temperature', result.columns)
|
|
133
|
+
self.assertIn('precipitation', result.columns)
|
|
134
|
+
|
|
135
|
+
# Test filtering
|
|
136
|
+
result = ctx.sql(
|
|
137
|
+
'SELECT * FROM weather WHERE temperature > 10 LIMIT 50'
|
|
138
|
+
).to_pandas()
|
|
139
|
+
|
|
140
|
+
self.assertGreater(len(result), 0)
|
|
141
|
+
# All temperatures should be > 10
|
|
142
|
+
self.assertTrue((result['temperature'] > 10).all())
|
|
143
|
+
|
|
144
|
+
def test_synthetic_dataset_aggregations(self):
|
|
145
|
+
"""Test aggregation queries on synthetic dataset."""
|
|
146
|
+
ctx = XarrayContext()
|
|
147
|
+
ctx.from_dataset('synthetic', self.synthetic)
|
|
148
|
+
|
|
149
|
+
# Test COUNT
|
|
150
|
+
result = ctx.sql(
|
|
151
|
+
'SELECT COUNT(*) as total_count FROM synthetic'
|
|
152
|
+
).to_pandas()
|
|
153
|
+
self.assertEqual(len(result), 1)
|
|
154
|
+
self.assertGreater(result['total_count'].iloc[0], 0)
|
|
155
|
+
|
|
156
|
+
# Test MIN, MAX, AVG
|
|
157
|
+
result = ctx.sql(
|
|
158
|
+
"""
|
|
159
|
+
SELECT
|
|
160
|
+
MIN(temperature) as min_temp,
|
|
161
|
+
MAX(temperature) as max_temp,
|
|
162
|
+
AVG(temperature) as avg_temp
|
|
163
|
+
FROM synthetic
|
|
164
|
+
"""
|
|
165
|
+
).to_pandas()
|
|
166
|
+
|
|
167
|
+
self.assertEqual(len(result), 1)
|
|
168
|
+
self.assertLess(result['min_temp'].iloc[0], result['max_temp'].iloc[0])
|
|
169
|
+
self.assertGreaterEqual(
|
|
170
|
+
result['avg_temp'].iloc[0], result['min_temp'].iloc[0]
|
|
171
|
+
)
|
|
172
|
+
self.assertLessEqual(result['avg_temp'].iloc[0], result['max_temp'].iloc[0])
|
|
173
|
+
|
|
174
|
+
def test_spatial_grouping(self):
|
|
175
|
+
"""Test spatial grouping queries."""
|
|
176
|
+
ctx = XarrayContext()
|
|
177
|
+
ctx.from_dataset('air', self.air_small)
|
|
178
|
+
|
|
179
|
+
# Group by spatial coordinates
|
|
180
|
+
result = ctx.sql(
|
|
181
|
+
"""
|
|
182
|
+
SELECT
|
|
183
|
+
lat, lon,
|
|
184
|
+
AVG(air) as avg_air,
|
|
185
|
+
COUNT(*) as time_count
|
|
186
|
+
FROM air
|
|
187
|
+
GROUP BY lat, lon
|
|
188
|
+
ORDER BY lat, lon
|
|
189
|
+
"""
|
|
190
|
+
).to_pandas()
|
|
191
|
+
|
|
192
|
+
expected_spatial_points = (
|
|
193
|
+
self.air_small.sizes['lat'] * self.air_small.sizes['lon']
|
|
194
|
+
)
|
|
195
|
+
self.assertEqual(len(result), expected_spatial_points)
|
|
196
|
+
|
|
197
|
+
# Each spatial point should have same number of time steps
|
|
198
|
+
self.assertTrue(
|
|
199
|
+
(result['time_count'] == self.air_small.sizes['time']).all()
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def test_temporal_filtering(self):
|
|
203
|
+
"""Test temporal filtering and grouping."""
|
|
204
|
+
ctx = XarrayContext()
|
|
205
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
206
|
+
|
|
207
|
+
# Get unique time values for filtering
|
|
208
|
+
all_data = ctx.sql(
|
|
209
|
+
'SELECT DISTINCT time FROM weather ORDER BY time'
|
|
210
|
+
).to_pandas()
|
|
211
|
+
if len(all_data) > 2:
|
|
212
|
+
mid_time = all_data['time'].iloc[len(all_data) // 2]
|
|
213
|
+
|
|
214
|
+
# Filter by time
|
|
215
|
+
result = ctx.sql(
|
|
216
|
+
f"""
|
|
217
|
+
SELECT COUNT(*) as count_after
|
|
218
|
+
FROM weather
|
|
219
|
+
WHERE time >= '{mid_time}'
|
|
220
|
+
"""
|
|
221
|
+
).to_pandas()
|
|
222
|
+
|
|
223
|
+
self.assertGreater(result['count_after'].iloc[0], 0)
|
|
224
|
+
|
|
225
|
+
def test_station_dataset_queries(self):
|
|
226
|
+
"""Test queries on 1D station dataset."""
|
|
227
|
+
ctx = XarrayContext()
|
|
228
|
+
ctx.from_dataset('stations', self.stations)
|
|
229
|
+
|
|
230
|
+
# Basic select
|
|
231
|
+
result = ctx.sql('SELECT * FROM stations ORDER BY elevation').to_pandas()
|
|
232
|
+
self.assertEqual(len(result), 5)
|
|
233
|
+
|
|
234
|
+
# Test filtering by elevation
|
|
235
|
+
result = ctx.sql(
|
|
236
|
+
'SELECT name, elevation FROM stations WHERE elevation > 300 ORDER BY elevation'
|
|
237
|
+
).to_pandas()
|
|
238
|
+
|
|
239
|
+
self.assertGreater(len(result), 0)
|
|
240
|
+
self.assertTrue((result['elevation'] > 300).all())
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class SqlJoinTestCase(unittest.TestCase):
|
|
244
|
+
"""Test joining tabular data with raster data using from_dataset."""
|
|
245
|
+
|
|
246
|
+
def setUp(self):
|
|
247
|
+
"""Set up datasets for join testing."""
|
|
248
|
+
# Create a small air temperature dataset
|
|
249
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
250
|
+
self.air_small = self.air.isel(
|
|
251
|
+
time=slice(0, 12), lat=slice(0, 5), lon=slice(0, 8)
|
|
252
|
+
).chunk({'time': 6})
|
|
253
|
+
|
|
254
|
+
# Create station metadata as a simple tabular dataset
|
|
255
|
+
# Use coordinates that overlap with air dataset
|
|
256
|
+
air_lats = self.air_small.lat.values
|
|
257
|
+
air_lons = self.air_small.lon.values
|
|
258
|
+
|
|
259
|
+
# Create stations at specific lat/lon points
|
|
260
|
+
self.stations = xr.Dataset(
|
|
261
|
+
{
|
|
262
|
+
'station_id': (['station'], [101, 102, 103]),
|
|
263
|
+
'lat': (['station'], [air_lats[0], air_lats[2], air_lats[4]]),
|
|
264
|
+
'lon': (['station'], [air_lons[1], air_lons[3], air_lons[5]]),
|
|
265
|
+
'elevation': (['station'], [100, 250, 500]),
|
|
266
|
+
'name': (['station'], ['Downtown', 'Airport', 'Mountain']),
|
|
267
|
+
}
|
|
268
|
+
).chunk({'station': 3})
|
|
269
|
+
|
|
270
|
+
# Create region lookup table
|
|
271
|
+
self.regions = xr.Dataset(
|
|
272
|
+
{
|
|
273
|
+
'region_id': (['region'], [1, 2, 3, 4]),
|
|
274
|
+
'region_name': (['region'], ['North', 'South', 'East', 'West']),
|
|
275
|
+
'min_lat': (['region'], [60, 30, 40, 40]),
|
|
276
|
+
'max_lat': (['region'], [90, 60, 80, 80]),
|
|
277
|
+
'min_lon': (['region'], [-180, -180, -90, -180]),
|
|
278
|
+
'max_lon': (['region'], [180, 180, 180, -90]),
|
|
279
|
+
}
|
|
280
|
+
).chunk({'region': 4})
|
|
281
|
+
|
|
282
|
+
def test_simple_cross_join(self):
|
|
283
|
+
"""Test cross join between raster and tabular data."""
|
|
284
|
+
ctx = XarrayContext()
|
|
285
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
286
|
+
ctx.from_dataset('stations', self.stations)
|
|
287
|
+
|
|
288
|
+
# Test separate queries first to ensure both datasets work
|
|
289
|
+
air_result = ctx.sql(
|
|
290
|
+
'SELECT COUNT(*) as air_count FROM air_data'
|
|
291
|
+
).to_pandas()
|
|
292
|
+
station_result = ctx.sql(
|
|
293
|
+
'SELECT COUNT(*) as station_count FROM stations'
|
|
294
|
+
).to_pandas()
|
|
295
|
+
|
|
296
|
+
self.assertGreater(air_result['air_count'].iloc[0], 0)
|
|
297
|
+
self.assertGreater(station_result['station_count'].iloc[0], 0)
|
|
298
|
+
|
|
299
|
+
# Test that we can query both datasets in the same context
|
|
300
|
+
# This demonstrates multi-dataset capability without complex joins
|
|
301
|
+
air_sample = ctx.sql('SELECT air FROM air_data LIMIT 5').to_pandas()
|
|
302
|
+
station_sample = ctx.sql(
|
|
303
|
+
'SELECT station_id FROM stations LIMIT 5'
|
|
304
|
+
).to_pandas()
|
|
305
|
+
|
|
306
|
+
self.assertGreater(len(air_sample), 0)
|
|
307
|
+
self.assertGreater(len(station_sample), 0)
|
|
308
|
+
self.assertIn('air', air_sample.columns)
|
|
309
|
+
self.assertIn('station_id', station_sample.columns)
|
|
310
|
+
|
|
311
|
+
def test_coordinate_based_join(self):
|
|
312
|
+
"""Test joining on coordinate proximity."""
|
|
313
|
+
ctx = XarrayContext()
|
|
314
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
315
|
+
ctx.from_dataset('stations', self.stations)
|
|
316
|
+
|
|
317
|
+
# First test a simple cross join to ensure datasets are compatible
|
|
318
|
+
result = ctx.sql(
|
|
319
|
+
"""
|
|
320
|
+
SELECT COUNT(*) as total_combinations
|
|
321
|
+
FROM air_data a
|
|
322
|
+
CROSS JOIN stations s
|
|
323
|
+
"""
|
|
324
|
+
).to_pandas()
|
|
325
|
+
|
|
326
|
+
self.assertGreater(result['total_combinations'].iloc[0], 0)
|
|
327
|
+
|
|
328
|
+
# Test a simpler join condition
|
|
329
|
+
result = ctx.sql(
|
|
330
|
+
"""
|
|
331
|
+
SELECT
|
|
332
|
+
COUNT(*) as match_count
|
|
333
|
+
FROM air_data a, stations s
|
|
334
|
+
WHERE s.station_id = 101
|
|
335
|
+
"""
|
|
336
|
+
).to_pandas()
|
|
337
|
+
|
|
338
|
+
self.assertGreater(result['match_count'].iloc[0], 0)
|
|
339
|
+
|
|
340
|
+
def test_region_classification_join(self):
|
|
341
|
+
"""Test joining with region classification."""
|
|
342
|
+
ctx = XarrayContext()
|
|
343
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
344
|
+
ctx.from_dataset('regions', self.regions)
|
|
345
|
+
|
|
346
|
+
# Test that both datasets can be queried independently
|
|
347
|
+
air_result = ctx.sql(
|
|
348
|
+
'SELECT COUNT(*) as air_count FROM air_data'
|
|
349
|
+
).to_pandas()
|
|
350
|
+
region_result = ctx.sql(
|
|
351
|
+
'SELECT COUNT(*) as region_count FROM regions'
|
|
352
|
+
).to_pandas()
|
|
353
|
+
|
|
354
|
+
self.assertGreater(air_result['air_count'].iloc[0], 0)
|
|
355
|
+
self.assertGreater(region_result['region_count'].iloc[0], 0)
|
|
356
|
+
|
|
357
|
+
# Test a simpler region-based query without complex joins
|
|
358
|
+
result = ctx.sql(
|
|
359
|
+
"""
|
|
360
|
+
SELECT
|
|
361
|
+
region_name,
|
|
362
|
+
min_lat,
|
|
363
|
+
max_lat
|
|
364
|
+
FROM regions
|
|
365
|
+
WHERE min_lat < 50
|
|
366
|
+
"""
|
|
367
|
+
).to_pandas()
|
|
368
|
+
|
|
369
|
+
self.assertGreater(len(result), 0)
|
|
370
|
+
self.assertIn('region_name', result.columns)
|
|
371
|
+
|
|
372
|
+
def test_multiple_dataset_aggregation(self):
|
|
373
|
+
"""Test aggregating across multiple datasets."""
|
|
374
|
+
ctx = XarrayContext()
|
|
375
|
+
ctx.from_dataset('air_data', self.air_small)
|
|
376
|
+
ctx.from_dataset('stations', self.stations)
|
|
377
|
+
|
|
378
|
+
# Get statistics by elevation bands using station data
|
|
379
|
+
result = ctx.sql(
|
|
380
|
+
"""
|
|
381
|
+
SELECT
|
|
382
|
+
CASE
|
|
383
|
+
WHEN s.elevation < 200 THEN 'Low'
|
|
384
|
+
WHEN s.elevation < 400 THEN 'Medium'
|
|
385
|
+
ELSE 'High'
|
|
386
|
+
END as elevation_band,
|
|
387
|
+
COUNT(DISTINCT s.station_id) as station_count,
|
|
388
|
+
COUNT(*) as air_measurements,
|
|
389
|
+
AVG(a.air) as avg_air
|
|
390
|
+
FROM air_data a
|
|
391
|
+
CROSS JOIN stations s
|
|
392
|
+
GROUP BY elevation_band
|
|
393
|
+
ORDER BY elevation_band
|
|
394
|
+
"""
|
|
395
|
+
).to_pandas()
|
|
396
|
+
|
|
397
|
+
self.assertGreater(len(result), 0)
|
|
398
|
+
self.assertIn('elevation_band', result.columns)
|
|
399
|
+
# Should have Low, Medium, High bands
|
|
400
|
+
self.assertGreaterEqual(len(result), 1)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class SqlOptimizationTestCase(unittest.TestCase):
|
|
404
|
+
"""Test DataFusion optimizations like column selection and filters."""
|
|
405
|
+
|
|
406
|
+
def setUp(self):
|
|
407
|
+
"""Set up dataset for optimization testing."""
|
|
408
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
409
|
+
self.air_medium = self.air.isel(
|
|
410
|
+
time=slice(0, 100), lat=slice(0, 20), lon=slice(0, 30)
|
|
411
|
+
).chunk({'time': 50})
|
|
412
|
+
|
|
413
|
+
# Create multi-variable dataset
|
|
414
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-02T00')
|
|
415
|
+
self.weather_medium = self.weather.isel(
|
|
416
|
+
time=slice(0, 12), lat=slice(0, 15), lon=slice(0, 20)
|
|
417
|
+
).chunk({'time': 6})
|
|
418
|
+
|
|
419
|
+
def test_column_projection(self):
|
|
420
|
+
"""Test that selecting specific columns works efficiently."""
|
|
421
|
+
ctx = XarrayContext()
|
|
422
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
423
|
+
|
|
424
|
+
# Select only specific columns
|
|
425
|
+
result = ctx.sql(
|
|
426
|
+
'SELECT lat, lon, temperature FROM weather LIMIT 100'
|
|
427
|
+
).to_pandas()
|
|
428
|
+
|
|
429
|
+
# Should only have the requested columns
|
|
430
|
+
expected_columns = {'lat', 'lon', 'temperature'}
|
|
431
|
+
actual_columns = set(result.columns)
|
|
432
|
+
self.assertEqual(expected_columns, actual_columns)
|
|
433
|
+
|
|
434
|
+
# Should not include precipitation, time, level, reference_time
|
|
435
|
+
unwanted_columns = {'precipitation', 'time', 'level', 'reference_time'}
|
|
436
|
+
self.assertTrue(unwanted_columns.isdisjoint(actual_columns))
|
|
437
|
+
|
|
438
|
+
def test_where_clause_filtering(self):
|
|
439
|
+
"""Test WHERE clause filtering optimization."""
|
|
440
|
+
ctx = XarrayContext()
|
|
441
|
+
ctx.from_dataset('air', self.air_medium)
|
|
442
|
+
|
|
443
|
+
# Test numeric filtering
|
|
444
|
+
result = ctx.sql(
|
|
445
|
+
'SELECT * FROM air WHERE air > 280 AND air < 290'
|
|
446
|
+
).to_pandas()
|
|
447
|
+
|
|
448
|
+
if len(result) > 0:
|
|
449
|
+
self.assertTrue((result['air'] > 280).all())
|
|
450
|
+
self.assertTrue((result['air'] < 290).all())
|
|
451
|
+
|
|
452
|
+
# Test coordinate filtering
|
|
453
|
+
result = ctx.sql(
|
|
454
|
+
'SELECT * FROM air WHERE lat > 50 AND lon < -100'
|
|
455
|
+
).to_pandas()
|
|
456
|
+
|
|
457
|
+
if len(result) > 0:
|
|
458
|
+
self.assertTrue((result['lat'] > 50).all())
|
|
459
|
+
self.assertTrue((result['lon'] < -100).all())
|
|
460
|
+
|
|
461
|
+
def test_limit_optimization(self):
|
|
462
|
+
"""Test LIMIT clause optimization."""
|
|
463
|
+
ctx = XarrayContext()
|
|
464
|
+
ctx.from_dataset('air', self.air_medium)
|
|
465
|
+
|
|
466
|
+
# Test small limits
|
|
467
|
+
for limit in [1, 5, 10, 50]:
|
|
468
|
+
result = ctx.sql(f'SELECT * FROM air LIMIT {limit}').to_pandas()
|
|
469
|
+
self.assertLessEqual(len(result), limit)
|
|
470
|
+
if (
|
|
471
|
+
limit
|
|
472
|
+
<= self.air_medium.sizes['time']
|
|
473
|
+
* self.air_medium.sizes['lat']
|
|
474
|
+
* self.air_medium.sizes['lon']
|
|
475
|
+
):
|
|
476
|
+
self.assertEqual(len(result), limit)
|
|
477
|
+
|
|
478
|
+
def test_order_by_optimization(self):
|
|
479
|
+
"""Test ORDER BY clause."""
|
|
480
|
+
ctx = XarrayContext()
|
|
481
|
+
ctx.from_dataset('air', self.air_medium)
|
|
482
|
+
|
|
483
|
+
# Test ordering by different columns
|
|
484
|
+
result = ctx.sql(
|
|
485
|
+
'SELECT lat, lon, air FROM air ORDER BY air DESC LIMIT 20'
|
|
486
|
+
).to_pandas()
|
|
487
|
+
|
|
488
|
+
if len(result) > 1:
|
|
489
|
+
# Should be in descending order
|
|
490
|
+
air_values = result['air'].values
|
|
491
|
+
self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
|
|
492
|
+
|
|
493
|
+
# Test ordering by coordinates
|
|
494
|
+
result = ctx.sql(
|
|
495
|
+
'SELECT lat, lon, air FROM air ORDER BY lat ASC, lon DESC LIMIT 20'
|
|
496
|
+
).to_pandas()
|
|
497
|
+
|
|
498
|
+
if len(result) > 1:
|
|
499
|
+
# Check lat is ascending
|
|
500
|
+
lat_values = result['lat'].values
|
|
501
|
+
lat_diffs = np.diff(lat_values)
|
|
502
|
+
# Allow for equal values (same lat, different lon)
|
|
503
|
+
self.assertTrue(np.all(lat_diffs >= 0))
|
|
504
|
+
|
|
505
|
+
def test_aggregation_pushdown(self):
|
|
506
|
+
"""Test aggregation optimization."""
|
|
507
|
+
ctx = XarrayContext()
|
|
508
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
509
|
+
|
|
510
|
+
# Test GROUP BY optimization
|
|
511
|
+
result = ctx.sql(
|
|
512
|
+
"""
|
|
513
|
+
SELECT
|
|
514
|
+
lat,
|
|
515
|
+
COUNT(*) as point_count,
|
|
516
|
+
AVG(temperature) as avg_temp,
|
|
517
|
+
MIN(precipitation) as min_precip,
|
|
518
|
+
MAX(precipitation) as max_precip
|
|
519
|
+
FROM weather
|
|
520
|
+
GROUP BY lat
|
|
521
|
+
ORDER BY lat
|
|
522
|
+
"""
|
|
523
|
+
).to_pandas()
|
|
524
|
+
|
|
525
|
+
# Should have one row per unique latitude
|
|
526
|
+
expected_lats = len(self.weather_medium.lat)
|
|
527
|
+
self.assertEqual(len(result), expected_lats)
|
|
528
|
+
|
|
529
|
+
# All aggregation columns should be present
|
|
530
|
+
expected_agg_cols = {'point_count', 'avg_temp', 'min_precip', 'max_precip'}
|
|
531
|
+
self.assertTrue(expected_agg_cols.issubset(set(result.columns)))
|
|
532
|
+
|
|
533
|
+
def test_complex_filter_optimization(self):
|
|
534
|
+
"""Test complex filtering with multiple conditions."""
|
|
535
|
+
ctx = XarrayContext()
|
|
536
|
+
ctx.from_dataset('weather', self.weather_medium)
|
|
537
|
+
|
|
538
|
+
# Complex WHERE clause with AND/OR
|
|
539
|
+
result = ctx.sql(
|
|
540
|
+
"""
|
|
541
|
+
SELECT lat, lon, temperature, precipitation
|
|
542
|
+
FROM weather
|
|
543
|
+
WHERE (temperature > 15 AND precipitation < 50)
|
|
544
|
+
OR (lat > 45 AND lon < -100)
|
|
545
|
+
LIMIT 100
|
|
546
|
+
"""
|
|
547
|
+
).to_pandas()
|
|
548
|
+
|
|
549
|
+
# Verify the complex condition
|
|
550
|
+
if len(result) > 0:
|
|
551
|
+
condition1 = (result['temperature'] > 15) & (result['precipitation'] < 50)
|
|
552
|
+
condition2 = (result['lat'] > 45) & (result['lon'] < -100)
|
|
553
|
+
combined = condition1 | condition2
|
|
554
|
+
self.assertTrue(combined.all())
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class SqlComplexQueryTestCase(unittest.TestCase):
|
|
558
|
+
"""Test complex SQL queries with advanced features."""
|
|
559
|
+
|
|
560
|
+
def setUp(self):
|
|
561
|
+
"""Set up datasets for complex query testing."""
|
|
562
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
563
|
+
self.air_small = self.air.isel(
|
|
564
|
+
time=slice(0, 50), lat=slice(0, 15), lon=slice(0, 20)
|
|
565
|
+
).chunk({'time': 25})
|
|
566
|
+
|
|
567
|
+
self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
|
|
568
|
+
self.weather_small = self.weather.isel(
|
|
569
|
+
time=slice(0, 8), lat=slice(0, 10), lon=slice(0, 12)
|
|
570
|
+
).chunk({'time': 4})
|
|
571
|
+
|
|
572
|
+
def test_subqueries(self):
|
|
573
|
+
"""Test subqueries and CTEs."""
|
|
574
|
+
ctx = XarrayContext()
|
|
575
|
+
ctx.from_dataset('air', self.air_small)
|
|
576
|
+
|
|
577
|
+
# Subquery to find above-average temperatures
|
|
578
|
+
result = ctx.sql(
|
|
579
|
+
"""
|
|
580
|
+
SELECT lat, lon, air
|
|
581
|
+
FROM air
|
|
582
|
+
WHERE air > (
|
|
583
|
+
SELECT AVG(air) FROM air
|
|
584
|
+
)
|
|
585
|
+
LIMIT 50
|
|
586
|
+
"""
|
|
587
|
+
).to_pandas()
|
|
588
|
+
|
|
589
|
+
if len(result) > 0:
|
|
590
|
+
# Get the average to verify
|
|
591
|
+
avg_result = ctx.sql('SELECT AVG(air) as avg_air FROM air').to_pandas()
|
|
592
|
+
avg_air = avg_result['avg_air'].iloc[0]
|
|
593
|
+
|
|
594
|
+
# All results should be above average
|
|
595
|
+
self.assertTrue((result['air'] > avg_air).all())
|
|
596
|
+
|
|
597
|
+
def test_window_functions(self):
|
|
598
|
+
"""Test window functions if supported by DataFusion."""
|
|
599
|
+
ctx = XarrayContext()
|
|
600
|
+
ctx.from_dataset('air', self.air_small)
|
|
601
|
+
|
|
602
|
+
try:
|
|
603
|
+
# Test ROW_NUMBER window function
|
|
604
|
+
result = ctx.sql(
|
|
605
|
+
"""
|
|
606
|
+
SELECT
|
|
607
|
+
lat, lon, air,
|
|
608
|
+
ROW_NUMBER() OVER (PARTITION BY lat ORDER BY air DESC) as rank_in_lat
|
|
609
|
+
FROM air
|
|
610
|
+
WHERE lat IN (
|
|
611
|
+
SELECT DISTINCT lat FROM air LIMIT 3
|
|
612
|
+
)
|
|
613
|
+
ORDER BY lat, rank_in_lat
|
|
614
|
+
LIMIT 30
|
|
615
|
+
"""
|
|
616
|
+
).to_pandas()
|
|
617
|
+
|
|
618
|
+
if len(result) > 0:
|
|
619
|
+
# Check that ranking works within each lat
|
|
620
|
+
for lat_val in result['lat'].unique():
|
|
621
|
+
lat_data = result[result['lat'] == lat_val].sort_values('rank_in_lat')
|
|
622
|
+
if len(lat_data) > 1:
|
|
623
|
+
# Air values should be in descending order within each lat
|
|
624
|
+
air_values = lat_data['air'].values
|
|
625
|
+
self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
|
|
626
|
+
|
|
627
|
+
except Exception:
|
|
628
|
+
# Window functions might not be supported, skip test
|
|
629
|
+
self.skipTest('Window functions not supported')
|
|
630
|
+
|
|
631
|
+
def test_case_statements(self):
|
|
632
|
+
"""Test CASE statements for conditional logic."""
|
|
633
|
+
ctx = XarrayContext()
|
|
634
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
635
|
+
|
|
636
|
+
result = ctx.sql(
|
|
637
|
+
"""
|
|
638
|
+
SELECT
|
|
639
|
+
lat, lon,
|
|
640
|
+
temperature,
|
|
641
|
+
CASE
|
|
642
|
+
WHEN temperature < 0 THEN 'Freezing'
|
|
643
|
+
WHEN temperature < 10 THEN 'Cold'
|
|
644
|
+
WHEN temperature < 20 THEN 'Cool'
|
|
645
|
+
WHEN temperature < 30 THEN 'Warm'
|
|
646
|
+
ELSE 'Hot'
|
|
647
|
+
END as temp_category,
|
|
648
|
+
precipitation,
|
|
649
|
+
CASE
|
|
650
|
+
WHEN precipitation < 5 THEN 'Dry'
|
|
651
|
+
WHEN precipitation < 20 THEN 'Light'
|
|
652
|
+
WHEN precipitation < 50 THEN 'Moderate'
|
|
653
|
+
ELSE 'Heavy'
|
|
654
|
+
END as precip_category
|
|
655
|
+
FROM weather
|
|
656
|
+
LIMIT 50
|
|
657
|
+
"""
|
|
658
|
+
).to_pandas()
|
|
659
|
+
|
|
660
|
+
self.assertGreater(len(result), 0)
|
|
661
|
+
self.assertIn('temp_category', result.columns)
|
|
662
|
+
self.assertIn('precip_category', result.columns)
|
|
663
|
+
|
|
664
|
+
# Verify categories are assigned correctly
|
|
665
|
+
for _, row in result.iterrows():
|
|
666
|
+
temp = row['temperature']
|
|
667
|
+
temp_cat = row['temp_category']
|
|
668
|
+
|
|
669
|
+
if temp < 0:
|
|
670
|
+
self.assertEqual(temp_cat, 'Freezing')
|
|
671
|
+
elif temp < 10:
|
|
672
|
+
self.assertEqual(temp_cat, 'Cold')
|
|
673
|
+
elif temp < 20:
|
|
674
|
+
self.assertEqual(temp_cat, 'Cool')
|
|
675
|
+
elif temp < 30:
|
|
676
|
+
self.assertEqual(temp_cat, 'Warm')
|
|
677
|
+
else:
|
|
678
|
+
self.assertEqual(temp_cat, 'Hot')
|
|
679
|
+
|
|
680
|
+
def test_mathematical_functions(self):
|
|
681
|
+
"""Test mathematical functions in SQL."""
|
|
682
|
+
ctx = XarrayContext()
|
|
683
|
+
ctx.from_dataset('weather', self.weather_small)
|
|
684
|
+
|
|
685
|
+
result = ctx.sql(
|
|
686
|
+
"""
|
|
687
|
+
SELECT
|
|
688
|
+
lat, lon,
|
|
689
|
+
temperature,
|
|
690
|
+
ROUND(temperature, 1) as temp_rounded,
|
|
691
|
+
ABS(temperature - 20) as temp_diff_from_20,
|
|
692
|
+
SQRT(ABS(temperature)) as temp_sqrt,
|
|
693
|
+
precipitation,
|
|
694
|
+
LOG(precipitation + 1) as log_precip
|
|
695
|
+
FROM weather
|
|
696
|
+
WHERE precipitation > 0
|
|
697
|
+
LIMIT 30
|
|
698
|
+
"""
|
|
699
|
+
).to_pandas()
|
|
700
|
+
|
|
701
|
+
if len(result) > 0:
|
|
702
|
+
# Verify mathematical operations
|
|
703
|
+
for _, row in result.iterrows():
|
|
704
|
+
temp = row['temperature']
|
|
705
|
+
self.assertAlmostEqual(
|
|
706
|
+
row['temp_diff_from_20'], abs(temp - 20), places=5
|
|
707
|
+
)
|
|
708
|
+
if temp >= 0:
|
|
709
|
+
self.assertAlmostEqual(row['temp_sqrt'], np.sqrt(temp), places=5)
|
|
710
|
+
|
|
711
|
+
def test_string_operations(self):
|
|
712
|
+
"""Test string operations if applicable."""
|
|
713
|
+
ctx = XarrayContext()
|
|
714
|
+
ctx.from_dataset('air', self.air_small)
|
|
715
|
+
|
|
716
|
+
# Test string operations on numeric data converted to string
|
|
717
|
+
result = ctx.sql(
|
|
718
|
+
"""
|
|
719
|
+
SELECT
|
|
720
|
+
lat, lon,
|
|
721
|
+
CAST(lat AS VARCHAR) as lat_str,
|
|
722
|
+
CONCAT('Lat: ', CAST(lat AS VARCHAR), ', Lon: ', CAST(lon AS VARCHAR)) as coordinates
|
|
723
|
+
FROM air
|
|
724
|
+
LIMIT 10
|
|
725
|
+
"""
|
|
726
|
+
).to_pandas()
|
|
727
|
+
|
|
728
|
+
self.assertGreater(len(result), 0)
|
|
729
|
+
self.assertIn('coordinates', result.columns)
|
|
730
|
+
|
|
731
|
+
# Verify concatenation worked
|
|
732
|
+
for _, row in result.iterrows():
|
|
733
|
+
expected = f"Lat: {row['lat']}, Lon: {row['lon']}"
|
|
734
|
+
# Allow for slight formatting differences
|
|
735
|
+
self.assertIn('Lat:', row['coordinates'])
|
|
736
|
+
self.assertIn('Lon:', row['coordinates'])
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
class SqlErrorHandlingTestCase(unittest.TestCase):
|
|
740
|
+
"""Test error handling and edge cases in SQL interface."""
|
|
741
|
+
|
|
742
|
+
def setUp(self):
|
|
743
|
+
"""Set up dataset for error testing."""
|
|
744
|
+
self.air = xr.tutorial.open_dataset('air_temperature')
|
|
745
|
+
self.air_small = self.air.isel(
|
|
746
|
+
time=slice(0, 10), lat=slice(0, 5), lon=slice(0, 8)
|
|
747
|
+
).chunk({'time': 5})
|
|
748
|
+
|
|
749
|
+
def test_invalid_table_name(self):
|
|
750
|
+
"""Test error handling for invalid table names."""
|
|
751
|
+
ctx = XarrayContext()
|
|
752
|
+
ctx.from_dataset('air', self.air_small)
|
|
753
|
+
|
|
754
|
+
with self.assertRaises(Exception):
|
|
755
|
+
ctx.sql('SELECT * FROM nonexistent_table')
|
|
756
|
+
|
|
757
|
+
def test_invalid_column_name(self):
|
|
758
|
+
"""Test error handling for invalid column names."""
|
|
759
|
+
ctx = XarrayContext()
|
|
760
|
+
ctx.from_dataset('air', self.air_small)
|
|
761
|
+
|
|
762
|
+
with self.assertRaises(Exception):
|
|
763
|
+
ctx.sql('SELECT nonexistent_column FROM air')
|
|
764
|
+
|
|
765
|
+
def test_syntax_errors(self):
|
|
766
|
+
"""Test handling of SQL syntax errors."""
|
|
767
|
+
ctx = XarrayContext()
|
|
768
|
+
ctx.from_dataset('air', self.air_small)
|
|
769
|
+
|
|
770
|
+
# Invalid SQL syntax
|
|
771
|
+
with self.assertRaises(Exception):
|
|
772
|
+
ctx.sql('SELECT * FORM air') # Typo: FORM instead of FROM
|
|
773
|
+
|
|
774
|
+
with self.assertRaises(Exception):
|
|
775
|
+
ctx.sql('SELECT * FROM air WHERE') # Incomplete WHERE clause
|
|
776
|
+
|
|
777
|
+
def test_empty_dataset_handling(self):
|
|
778
|
+
"""Test handling of empty datasets."""
|
|
779
|
+
# Create empty dataset
|
|
780
|
+
empty_ds = xr.Dataset({'temp': (['x'], [])}, coords={'x': []}).chunk(
|
|
781
|
+
{'x': 1}
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
ctx = XarrayContext()
|
|
785
|
+
ctx.from_dataset('empty', empty_ds)
|
|
786
|
+
|
|
787
|
+
# Should handle empty dataset gracefully
|
|
788
|
+
result = ctx.sql('SELECT * FROM empty').to_pandas()
|
|
789
|
+
self.assertEqual(len(result), 0)
|
|
790
|
+
|
|
791
|
+
# Aggregations on empty dataset
|
|
792
|
+
result = ctx.sql('SELECT COUNT(*) as count FROM empty').to_pandas()
|
|
793
|
+
self.assertEqual(len(result), 1)
|
|
794
|
+
self.assertEqual(result['count'].iloc[0], 0)
|
|
795
|
+
|
|
796
|
+
def test_large_limit_handling(self):
|
|
797
|
+
"""Test handling of very large LIMIT values."""
|
|
798
|
+
ctx = XarrayContext()
|
|
799
|
+
ctx.from_dataset('air', self.air_small)
|
|
800
|
+
|
|
801
|
+
total_rows = (
|
|
802
|
+
self.air_small.sizes['time']
|
|
803
|
+
* self.air_small.sizes['lat']
|
|
804
|
+
* self.air_small.sizes['lon']
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
# Request more rows than exist
|
|
808
|
+
result = ctx.sql(f'SELECT * FROM air LIMIT {total_rows * 10}').to_pandas()
|
|
809
|
+
self.assertEqual(len(result), total_rows)
|
|
810
|
+
|
|
811
|
+
|
|
63
812
|
if __name__ == '__main__':
|
|
64
813
|
unittest.main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Querry Xarray with SQL.
|
|
5
5
|
Author-email: Alexander Merose <al@merose.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -46,19 +46,35 @@ pip install xarray-sql
|
|
|
46
46
|
|
|
47
47
|
## What is this?
|
|
48
48
|
|
|
49
|
-
This is an experiment to provide a SQL interface for
|
|
49
|
+
This is an experiment to provide a SQL interface for array datasets.
|
|
50
50
|
|
|
51
51
|
```python
|
|
52
52
|
import xarray as xr
|
|
53
|
-
import xarray_sql as
|
|
53
|
+
import xarray_sql as xql
|
|
54
54
|
|
|
55
55
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
56
56
|
|
|
57
57
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
ctx = xql.XarrayContext()
|
|
59
|
+
ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
|
|
60
|
+
# DataFrame()
|
|
61
|
+
# +------+---------------------+-------+--------------------+
|
|
62
|
+
# | lat | time | lon | air |
|
|
63
|
+
# +------+---------------------+-------+--------------------+
|
|
64
|
+
# | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
|
|
65
|
+
# | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
|
|
66
|
+
# | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
|
|
67
|
+
# | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
|
|
68
|
+
# | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
|
|
69
|
+
# | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
|
|
70
|
+
# | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
|
|
71
|
+
# | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
|
|
72
|
+
# | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
|
|
73
|
+
# | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
|
|
74
|
+
# +------+---------------------+-------+--------------------+
|
|
75
|
+
# Data truncated.
|
|
76
|
+
|
|
77
|
+
result = ctx.sql('''
|
|
62
78
|
SELECT
|
|
63
79
|
"lat", "lon", AVG("air") as air_total
|
|
64
80
|
FROM
|
|
@@ -66,16 +82,36 @@ df = c.sql('''
|
|
|
66
82
|
GROUP BY
|
|
67
83
|
"lat", "lon"
|
|
68
84
|
''')
|
|
85
|
+
# DataFrame()
|
|
86
|
+
# +------+-------+--------------------+
|
|
87
|
+
# | lat | lon | air_total |
|
|
88
|
+
# +------+-------+--------------------+
|
|
89
|
+
# | 75.0 | 205.0 | 259.88662671232834 |
|
|
90
|
+
# | 75.0 | 207.5 | 259.48268150684896 |
|
|
91
|
+
# | 75.0 | 230.0 | 258.9192123287667 |
|
|
92
|
+
# | 75.0 | 275.0 | 257.07574315068456 |
|
|
93
|
+
# | 75.0 | 322.5 | 250.11792123287654 |
|
|
94
|
+
# | 75.0 | 325.0 | 250.81590068493125 |
|
|
95
|
+
# | 72.5 | 205.0 | 262.74933904109537 |
|
|
96
|
+
# | 72.5 | 207.5 | 262.5384315068488 |
|
|
97
|
+
# | 72.5 | 230.0 | 260.8287945205475 |
|
|
98
|
+
# | 72.5 | 275.0 | 257.30633219178037 |
|
|
99
|
+
# +------+-------+--------------------+
|
|
100
|
+
# Data truncated.
|
|
101
|
+
#
|
|
69
102
|
|
|
70
103
|
# A table of the average temperature for each location across time.
|
|
71
|
-
df.to_pandas()
|
|
72
|
-
|
|
73
|
-
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
74
|
-
df = qr.read_xarray(ds).to_pandas()
|
|
104
|
+
df = result.to_pandas()
|
|
75
105
|
df.head()
|
|
106
|
+
# lat lon air_total
|
|
107
|
+
# 0 75.0 210.0 259.016562
|
|
108
|
+
# 1 75.0 222.5 258.362212
|
|
109
|
+
# 2 75.0 237.5 258.318240
|
|
110
|
+
# 3 75.0 267.5 256.928497
|
|
111
|
+
# 4 75.0 285.0 261.614103
|
|
76
112
|
```
|
|
77
113
|
|
|
78
|
-
Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
|
|
114
|
+
Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
|
|
79
115
|
SQL queries against them.
|
|
80
116
|
|
|
81
117
|
## Why build this?
|
|
@@ -93,18 +129,18 @@ A few reasons:
|
|
|
93
129
|
|
|
94
130
|
This is a light-weight way to prove the value of the interface.
|
|
95
131
|
|
|
96
|
-
The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
|
|
97
|
-
is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
98
|
-
front end.
|
|
132
|
+
The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
|
|
133
|
+
ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
134
|
+
DB front end.
|
|
99
135
|
|
|
100
136
|
## How does it work?
|
|
101
137
|
|
|
102
|
-
All chunks in
|
|
138
|
+
All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
|
|
103
139
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
104
140
|
That's it!
|
|
105
141
|
|
|
106
|
-
_2025 update_: This library now implements a
|
|
107
|
-
pure
|
|
142
|
+
_2025 update_: This library now implements a Dask-like `from_map` interface in
|
|
143
|
+
pure DataFusion and PyArrow, but works with the same principle!
|
|
108
144
|
|
|
109
145
|
## Why does this work?
|
|
110
146
|
|
|
@@ -112,14 +148,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
|
112
148
|
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
113
149
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
114
150
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
115
|
-
amenable to a DataFrame.
|
|
116
|
-
|
|
117
|
-
There is added overhead from duplicating dimensions as columns, which we see as
|
|
118
|
-
worth the convenience of DataFrames.
|
|
151
|
+
amenable to a DataFrame. We take advantage of this light weight metadata change to
|
|
152
|
+
make chunked information scannable by a DB engine (DataFusion).
|
|
119
153
|
|
|
120
154
|
## What are the current limitations?
|
|
121
155
|
|
|
122
|
-
_2025 update_: TBD,
|
|
156
|
+
_2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
|
|
157
|
+
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
158
|
+
project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
|
|
159
|
+
you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
|
|
123
160
|
|
|
124
161
|
## What would a deeper integration look like?
|
|
125
162
|
|
|
@@ -134,6 +171,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
|
134
171
|
and BigQuery. More thoughts on this
|
|
135
172
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
136
173
|
|
|
174
|
+
_2025 update_: Something like this is being built across a few projects! The ones I know about are:
|
|
175
|
+
- [CartoDB's Raquet](https://github.com/CartoDB/raquet)
|
|
176
|
+
- The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
|
|
177
|
+
|
|
178
|
+
As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
|
|
179
|
+
rust-based DataFusion backend provided by arrow-zarr.
|
|
180
|
+
|
|
137
181
|
## Sponsors & Contributors
|
|
138
182
|
|
|
139
183
|
I want to give a special thanks to the following folks and institutions:
|
|
@@ -143,6 +187,9 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
143
187
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
144
188
|
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
145
189
|
for the early review and discussion of this project.
|
|
190
|
+
- Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
|
|
191
|
+
Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
|
|
192
|
+
and DataFusion-specific collaboration.
|
|
146
193
|
|
|
147
194
|
## License
|
|
148
195
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
|
|
2
2
|
xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
|
|
3
|
-
xarray_sql/df.py,sha256=
|
|
3
|
+
xarray_sql/df.py,sha256=1BmcA8mq59er_6bxsxxZnHQ3zyMEeXoBblQtj-eVTLU,6065
|
|
4
4
|
xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
|
|
5
5
|
xarray_sql/df_test.py,sha256=w2GqqaD2iUlIPA9jkyzUlHJJsFhQoHZB5pgtfz8xFgs,11965
|
|
6
6
|
xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
|
|
7
|
-
xarray_sql/sql_test.py,sha256=
|
|
8
|
-
xarray_sql-0.1.
|
|
9
|
-
xarray_sql-0.1.
|
|
10
|
-
xarray_sql-0.1.
|
|
11
|
-
xarray_sql-0.1.
|
|
12
|
-
xarray_sql-0.1.
|
|
7
|
+
xarray_sql/sql_test.py,sha256=Iy_w7EWSCTqQJuXa04WXLg-e05cZ6XL8DQVCsaBuXhA,24944
|
|
8
|
+
xarray_sql-0.1.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
9
|
+
xarray_sql-0.1.2.dist-info/METADATA,sha256=q7EdLLIRJH70cjG3vKNMiUCAGuj0afSQrKvj_fQIqss,8667
|
|
10
|
+
xarray_sql-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
11
|
+
xarray_sql-0.1.2.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
|
|
12
|
+
xarray_sql-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|