xarray-sql 0.2.0__cp313-cp313-manylinux_2_28_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ from .reader import read_xarray, read_xarray_table
2
+ from .sql import XarrayContext
3
+ from .df import from_map
4
+
5
+ __all__ = [
6
+ "XarrayContext",
7
+ "read_xarray_table",
8
+ "read_xarray",
9
+ "from_map", # deprecated
10
+ ]
@@ -0,0 +1,4 @@
1
+ [diffend] Oversized file quarantined before diffing.
2
+ name: xarray_sql/_native.cpython-313-x86_64-linux-gnu.so
3
+ size: 83702088 bytes
4
+ sha256: 4da1c660c73ba3582f16ee547e82d7f76c170942157a71af3bba8ef40f7cb9b1
xarray_sql/core.py ADDED
@@ -0,0 +1,48 @@
1
+ import itertools
2
+ import typing as t
3
+
4
+ import numpy as np
5
+ import xarray as xr
6
+
7
+ Row = t.List[t.Any]
8
+
9
+
10
+ # deprecated
11
+ def get_columns(ds: xr.Dataset) -> t.List[str]:
12
+ return list(ds.sizes.keys()) + list(ds.data_vars.keys())
13
+
14
+
15
+ # Deprecated
16
+ def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
17
+ dim_keys, dim_vals = zip(*ds.sizes.items())
18
+
19
+ for idx in itertools.product(*(range(d) for d in dim_vals)):
20
+ coord_idx = dict(zip(dim_keys, idx))
21
+ data = ds.isel(coord_idx)
22
+ coord_data = [ds.coords[v][coord_idx[v]] for v in dim_keys]
23
+ row = [v.values for v in coord_data + list(data.data_vars.values())]
24
+ yield row
25
+
26
+
27
+ # Deprecated
28
+ def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
29
+ """Unravel with unbounded memory (as a NumPy Array)."""
30
+ dim_keys, dim_vals = zip(*ds.sizes.items())
31
+ columns = get_columns(ds)
32
+
33
+ N = np.prod([d for d in dim_vals])
34
+
35
+ out = np.recarray((N,), dtype=[(c, ds[c].dtype) for c in columns])
36
+
37
+ for name, da in ds.items():
38
+ out[name] = da.values.ravel()
39
+
40
+ prod_vals = (ds.coords[k].values for k in dim_keys)
41
+ coords = np.array(np.meshgrid(*prod_vals), dtype=int).T.reshape(
42
+ -1, len(dim_keys)
43
+ )
44
+
45
+ for i, d in enumerate(dim_keys):
46
+ out[d] = coords[:, i]
47
+
48
+ return out
xarray_sql/df.py ADDED
@@ -0,0 +1,172 @@
1
+ import itertools
2
+ import typing as t
3
+ import warnings
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+ import pyarrow as pa
8
+ import xarray as xr
9
+ from datafusion.context import ArrowStreamExportable
10
+
11
+ Block = t.Dict[t.Hashable, slice]
12
+ Chunks = t.Optional[t.Dict[str, int]]
13
+
14
+
15
+ # Borrowed from Xarray
16
+ def _get_chunk_slicer(
17
+ dim: t.Hashable, chunk_index: t.Mapping, chunk_bounds: t.Mapping
18
+ ):
19
+ if dim in chunk_index:
20
+ which_chunk = chunk_index[dim]
21
+ return slice(
22
+ chunk_bounds[dim][which_chunk], chunk_bounds[dim][which_chunk + 1]
23
+ )
24
+ return slice(None)
25
+
26
+
27
+ # Adapted from Xarray `map_blocks` implementation.
28
+ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
29
+ """Compute block slices for a chunked Dataset."""
30
+ if chunks is not None:
31
+ for_chunking = ds.copy(data=None, deep=False).chunk(chunks)
32
+ chunks = for_chunking.chunks
33
+ del for_chunking
34
+ else:
35
+ chunks = ds.chunks
36
+
37
+ assert chunks, "Dataset `ds` must be chunked or `chunks` must be provided."
38
+
39
+ # chunks is Dict[str, Tuple[int, ...]] from xarray
40
+ chunk_bounds = {
41
+ dim: np.cumsum((0,) + tuple(c)) # type: ignore[arg-type]
42
+ for dim, c in chunks.items()
43
+ }
44
+ ichunk = {dim: range(len(tuple(c))) for dim, c in chunks.items()} # type: ignore[arg-type]
45
+ ick, icv = zip(*ichunk.items()) # Makes same order of keys and val.
46
+ chunk_idxs = (dict(zip(ick, i)) for i in itertools.product(*icv))
47
+ blocks = (
48
+ {
49
+ dim: _get_chunk_slicer(dim, chunk_index, chunk_bounds)
50
+ for dim in ds.dims
51
+ }
52
+ for chunk_index in chunk_idxs
53
+ )
54
+ yield from blocks
55
+
56
+
57
+ def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
58
+ """Explodes a dataset into its chunks."""
59
+ yield from (ds.isel(b) for b in block_slices(ds, chunks=chunks))
60
+
61
+
62
+ def _block_len(block: Block) -> int:
63
+ return int(np.prod([v.stop - v.start for v in block.values()]))
64
+
65
+
66
+ def from_map_batched(
67
+ func: t.Callable[..., pd.DataFrame],
68
+ *iterables,
69
+ args: t.Optional[t.Tuple] = None,
70
+ schema: pa.Schema = None,
71
+ **kwargs,
72
+ ) -> pa.RecordBatchReader:
73
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
74
+
75
+ This is equivalent to dask's from_map but returns a PyArrow
76
+ RecordBatchReader that can be used with DataFusion. It iterates over
77
+ RecordBatches which are created via the `func` one-at-a-time.
78
+
79
+ Args:
80
+ func: Function to apply to each element of the iterables. Currently, the function
81
+ must return a Pandas DataFrame.
82
+ *iterables: Iterable objects to map the function over.
83
+ schema: Optional schema needed for the RecordBatchReader.
84
+ args: Additional positional arguments to pass to func.
85
+ **kwargs: Additional keyword arguments to pass to func.
86
+
87
+ Returns:
88
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
89
+ """
90
+ if args is None:
91
+ args = ()
92
+
93
+ def map_batches():
94
+ for items in zip(*iterables):
95
+ df = func(*items, *args, **kwargs)
96
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
97
+
98
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
99
+
100
+
101
+ def from_map(
102
+ func: t.Callable, *iterables, args: t.Optional[t.Tuple] = None, **kwargs
103
+ ) -> pa.Table:
104
+ """Create a PyArrow Table by mapping a function over iterables.
105
+
106
+ This is equivalent to dask's from_map but returns a PyArrow Table
107
+ that can be used with DataFusion instead of a Dask DataFrame.
108
+
109
+ Args:
110
+ func: Function to apply to each element of the iterables.
111
+ *iterables: Iterable objects to map the function over.
112
+ args: Additional positional arguments to pass to func.
113
+ **kwargs: Additional keyword arguments to pass to func.
114
+
115
+ Returns:
116
+ A PyArrow Table containing the concatenated results.
117
+ """
118
+ if args is None:
119
+ args = ()
120
+
121
+ # Apply the function to each combination of iterable elements
122
+ results = []
123
+ for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
124
+ if isinstance(items, tuple):
125
+ result = func(*items, *args, **kwargs)
126
+ else:
127
+ result = func(items, *args, **kwargs)
128
+
129
+ # Convert result to PyArrow Table
130
+ if isinstance(result, pd.DataFrame):
131
+ pa_table = pa.Table.from_pandas(result)
132
+ elif isinstance(result, pa.Table):
133
+ pa_table = result
134
+ else:
135
+ # Try to convert to pandas first, then to PyArrow
136
+ try:
137
+ df = pd.DataFrame(result)
138
+ pa_table = pa.Table.from_pandas(df)
139
+ except Exception as e:
140
+ raise ValueError(
141
+ f"Cannot convert function result to PyArrow Table: {e}"
142
+ )
143
+
144
+ results.append(pa_table)
145
+
146
+ # Concatenate all results
147
+ if not results:
148
+ raise ValueError("No results to concatenate")
149
+
150
+ return pa.concat_tables(results)
151
+
152
+
153
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
154
+ """Converts an xarray Dataset to a pandas DataFrame."""
155
+ return ds.to_dataframe().reset_index() # type: ignore[no-any-return]
156
+
157
+
158
+ def _parse_schema(ds) -> pa.Schema:
159
+ """Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
160
+ columns = []
161
+
162
+ for coord_name, coord_var in ds.coords.items():
163
+ # Only include dimension coordinates
164
+ if coord_name in ds.dims:
165
+ pa_type = pa.from_numpy_dtype(coord_var.dtype)
166
+ columns.append(pa.field(coord_name, pa_type))
167
+
168
+ for var_name, var in ds.data_vars.items():
169
+ pa_type = pa.from_numpy_dtype(var.dtype)
170
+ columns.append(pa.field(var_name, pa_type))
171
+
172
+ return pa.schema(columns)
xarray_sql/df_test.py ADDED
@@ -0,0 +1,341 @@
1
+ import itertools
2
+ import tracemalloc
3
+
4
+ import numpy as np
5
+ import pandas as pd
6
+ import pyarrow as pa
7
+ import pytest
8
+ import xarray as xr
9
+
10
+ from .reader import read_xarray
11
+ from .df import explode, block_slices, from_map, pivot, from_map_batched
12
+
13
+
14
+ def rand_wx(start: str, end: str) -> xr.Dataset:
15
+ np.random.seed(42)
16
+ lat = np.linspace(-90, 90, num=720)
17
+ lon = np.linspace(-180, 180, num=1440)
18
+ time = pd.date_range(start, end, freq="h")
19
+ level = np.array([1000, 500], dtype=np.int32)
20
+ reference_time = pd.Timestamp(start)
21
+ temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
22
+ precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
23
+ return xr.Dataset(
24
+ data_vars=dict(
25
+ temperature=(["lat", "lon", "time", "level"], temperature),
26
+ precipitation=(["lat", "lon", "time", "level"], precipitation),
27
+ ),
28
+ coords=dict(
29
+ lat=lat,
30
+ lon=lon,
31
+ time=time,
32
+ level=level,
33
+ reference_time=reference_time,
34
+ ),
35
+ attrs=dict(description="Random weather."),
36
+ )
37
+
38
+
39
+ def create_large_dataset(time_steps=1000, lat_points=100, lon_points=100):
40
+ """Create a large xarray dataset for memory testing."""
41
+ np.random.seed(42)
42
+
43
+ time = pd.date_range("2020-01-01", periods=time_steps, freq="h")
44
+ lat = np.linspace(-90, 90, lat_points)
45
+ lon = np.linspace(-180, 180, lon_points)
46
+
47
+ temp_data = np.random.rand(time_steps, lat_points, lon_points) * 40 - 10
48
+ precip_data = np.random.rand(time_steps, lat_points, lon_points) * 100
49
+
50
+ return xr.Dataset(
51
+ {
52
+ "temperature": (["time", "lat", "lon"], temp_data),
53
+ "precipitation": (["time", "lat", "lon"], precip_data),
54
+ },
55
+ coords={"time": time, "lat": lat, "lon": lon},
56
+ )
57
+
58
+
59
+ def adding_function(x, y):
60
+ """Simple function that adds two values and returns a DataFrame."""
61
+ result = pd.DataFrame({"x": [x], "y": [y], "sum": [x + y]})
62
+ return result
63
+
64
+
65
+ @pytest.fixture
66
+ def air():
67
+ ds = xr.tutorial.open_dataset("air_temperature")
68
+ chunks = {"time": 240}
69
+ return ds.chunk(chunks)
70
+
71
+
72
+ @pytest.fixture
73
+ def air_small(air):
74
+ return air.isel(time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)).chunk(
75
+ {"time": 240}
76
+ )
77
+
78
+
79
+ @pytest.fixture
80
+ def randwx():
81
+ return rand_wx("1995-01-13T00", "1995-01-13T01")
82
+
83
+
84
+ @pytest.fixture
85
+ def large_ds():
86
+ return create_large_dataset().chunk({"time": 25})
87
+
88
+
89
+ def test_explode_cardinality(air):
90
+ dss = explode(air)
91
+ assert len(list(dss)) == np.prod([len(c) for c in air.chunks.values()])
92
+
93
+
94
+ def test_explode_dim_sizes_one(air):
95
+ chunks = {"time": 240}
96
+ ds = next(iter(explode(air)))
97
+ for k, v in chunks.items():
98
+ assert k in ds.dims
99
+ assert v == ds.sizes[k]
100
+
101
+
102
+ @pytest.mark.skip(reason="TODO(alxmrs): Why is this test slow?")
103
+ def test_explode_dim_sizes_all(air):
104
+ dss = explode(air)
105
+ assert [tuple(ds.dims.values()) for ds in dss] == list(
106
+ itertools.product(*air.chunksizes.values())
107
+ )
108
+
109
+
110
+ def test_explode_data_equal_one_first(air):
111
+ ds = next(iter(explode(air)))
112
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
113
+ assert air.isel(iselection).equals(ds)
114
+
115
+
116
+ def test_explode_data_equal_one_last(air):
117
+ dss = list(explode(air))
118
+ ds = dss[-1]
119
+
120
+ # For the last chunk, we need to calculate where it actually starts
121
+ # The original logic slice(0, s) only works for the first chunk
122
+ iselection = {}
123
+ for dim in ds.dims:
124
+ # Get chunk boundaries
125
+ chunk_bounds = np.cumsum((0,) + air.chunks[dim])
126
+ # Last chunk index
127
+ last_chunk_idx = len(air.chunks[dim]) - 1
128
+ # Calculate actual start and end positions
129
+ start = chunk_bounds[last_chunk_idx]
130
+ end = chunk_bounds[last_chunk_idx + 1]
131
+ iselection[dim] = slice(start, end)
132
+
133
+ assert air.isel(iselection).equals(ds)
134
+
135
+
136
+ def test_from_map_basic():
137
+ def make_df(x):
138
+ return pd.DataFrame({"value": [x, x * 2], "index": [0, 1]})
139
+
140
+ result = from_map(make_df, [1, 2, 3])
141
+ assert isinstance(result, pa.Table)
142
+ assert len(result) == 6
143
+ assert result.column_names == ["value", "index"]
144
+
145
+
146
+ def test_from_map_multiple_iterables():
147
+ def add_values(x, y):
148
+ return pd.DataFrame({"sum": [x + y], "x": [x], "y": [y]})
149
+
150
+ result = from_map(add_values, [1, 2], [10, 20])
151
+ assert isinstance(result, pa.Table)
152
+ assert len(result) == 2
153
+
154
+ df = result.to_pandas()
155
+ assert list(df["sum"]) == [11, 22]
156
+
157
+
158
+ def test_from_map_with_args():
159
+ def multiply_and_add(x, multiplier, add_value):
160
+ return pd.DataFrame({"result": [x * multiplier + add_value]})
161
+
162
+ result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
163
+ assert isinstance(result, pa.Table)
164
+ assert len(result) == 3
165
+
166
+ df = result.to_pandas()
167
+ assert list(df["result"]) == [12, 14, 16]
168
+
169
+
170
+ def test_from_map_with_pyarrow_tables():
171
+ def make_arrow_table(x):
172
+ df = pd.DataFrame({"value": [x]})
173
+ return pa.Table.from_pandas(df)
174
+
175
+ result = from_map(make_arrow_table, [1, 2, 3])
176
+ assert isinstance(result, pa.Table)
177
+ assert len(result) == 3
178
+
179
+
180
+ def test_from_map_batched_basic_functionality(air_small):
181
+ blocks = list(block_slices(air_small, chunks={"time": 4, "lat": 3, "lon": 4}))
182
+
183
+ first_block_df = pivot(air_small.isel(blocks[0]))
184
+ expected_schema = pa.Schema.from_pandas(first_block_df)
185
+
186
+ reader = from_map_batched(
187
+ pivot, [air_small.isel(block) for block in blocks], schema=expected_schema
188
+ )
189
+
190
+ assert isinstance(reader, pa.RecordBatchReader)
191
+ assert reader.schema == expected_schema
192
+
193
+ batches = list(reader)
194
+ assert len(batches) > 0
195
+ for batch in batches:
196
+ assert batch.schema == expected_schema
197
+ assert len(batch) > 0
198
+
199
+
200
+ def test_from_map_batched_multiple_iterables():
201
+ x_values = [1, 2, 3, 4, 5]
202
+ y_values = [10, 20, 30, 40, 50]
203
+
204
+ expected_schema = pa.schema(
205
+ [("x", pa.int64()), ("y", pa.int64()), ("sum", pa.int64())]
206
+ )
207
+
208
+ reader = from_map_batched(
209
+ adding_function, x_values, y_values, schema=expected_schema
210
+ )
211
+ table = reader.read_all()
212
+ df = table.to_pandas()
213
+
214
+ expected_df = pd.DataFrame(
215
+ {
216
+ "x": x_values,
217
+ "y": y_values,
218
+ "sum": [x + y for x, y in zip(x_values, y_values)],
219
+ }
220
+ )
221
+ pd.testing.assert_frame_equal(df, expected_df)
222
+
223
+
224
+ def test_from_map_batched_with_args_and_kwargs():
225
+ def multiply_and_add(x, multiplier, offset=0):
226
+ return pd.DataFrame({"x": [x], "result": [x * multiplier + offset]})
227
+
228
+ values = [1, 2, 3]
229
+ expected_schema = pa.schema([("x", pa.int64()), ("result", pa.int64())])
230
+
231
+ reader = from_map_batched(
232
+ multiply_and_add, values, args=(2,), offset=5, schema=expected_schema
233
+ )
234
+ table = reader.read_all()
235
+ df = table.to_pandas()
236
+
237
+ expected_df = pd.DataFrame({"x": [1, 2, 3], "result": [7, 9, 11]})
238
+ pd.testing.assert_frame_equal(df, expected_df)
239
+
240
+
241
+ def test_from_map_batched_empty_iterables():
242
+ empty_schema = pa.schema([("value", pa.int64())])
243
+
244
+ reader = from_map_batched(
245
+ lambda x: pd.DataFrame({"value": [x]}), [], schema=empty_schema
246
+ )
247
+ batches = list(reader)
248
+ assert len(batches) == 0
249
+
250
+
251
+ def test_from_map_batched_consistency_with_regular_map(air_small):
252
+ blocks = list(block_slices(air_small, chunks={"time": 4, "lat": 3}))
253
+ datasets = [air_small.isel(block) for block in blocks]
254
+
255
+ first_df = pivot(datasets[0])
256
+ schema = pa.Schema.from_pandas(first_df)
257
+
258
+ reader = from_map_batched(pivot, datasets, schema=schema)
259
+ batched_table = reader.read_all()
260
+
261
+ regular_dfs = [pivot(ds) for ds in datasets]
262
+ regular_table = pa.Table.from_pandas(
263
+ pd.concat(regular_dfs, ignore_index=True)
264
+ )
265
+
266
+ assert batched_table.schema == regular_table.schema
267
+ assert len(batched_table) == len(regular_table)
268
+
269
+ batched_df = (
270
+ batched_table.to_pandas()
271
+ .sort_values(["time", "lat", "lon"])
272
+ .reset_index(drop=True)
273
+ )
274
+ regular_df = (
275
+ regular_table.to_pandas()
276
+ .sort_values(["time", "lat", "lon"])
277
+ .reset_index(drop=True)
278
+ )
279
+
280
+ pd.testing.assert_frame_equal(batched_df, regular_df)
281
+
282
+
283
+ def test_from_map_batched_integration_with_datafusion_via_read_xarray():
284
+ air = xr.tutorial.open_dataset("air_temperature")
285
+ air_small = air.isel(time=slice(0, 50), lat=slice(0, 10), lon=slice(0, 15))
286
+ air_chunked = air_small.chunk({"time": 25, "lat": 5, "lon": 8})
287
+
288
+ arrow_stream = read_xarray(
289
+ air_chunked, chunks={"time": 25, "lat": 5, "lon": 8}
290
+ )
291
+
292
+ assert hasattr(arrow_stream, "schema")
293
+ assert hasattr(arrow_stream, "__iter__")
294
+
295
+ table = arrow_stream.read_all()
296
+ assert len(table) > 0
297
+
298
+ expected_columns = {"time", "lat", "lon", "air"}
299
+ actual_columns = set(table.column_names)
300
+ assert expected_columns.issubset(actual_columns)
301
+
302
+
303
+ def test_read_xarray_loads_one_chunk_at_a_time(large_ds):
304
+ tracemalloc.start()
305
+ iterable = read_xarray(large_ds)
306
+ first_size, first_peak = tracemalloc.get_traced_memory()
307
+ tracemalloc.reset_peak()
308
+
309
+ sizes, peaks = [], []
310
+
311
+ first_chunk = large_ds.isel(next(block_slices(large_ds)))
312
+ chunk_size = first_chunk.nbytes
313
+
314
+ # Creating the iterator should be inexpensive -- less than one chunk.
315
+ # We multiply by constant factors because chunks have additional overhead
316
+ assert first_size < chunk_size * 3
317
+ assert first_peak < chunk_size * 6
318
+
319
+ for it in iterable:
320
+ _ = it
321
+ cur_size, cur_peak = tracemalloc.get_traced_memory()
322
+ tracemalloc.reset_peak()
323
+ sizes.append(cur_size)
324
+ peaks.append(cur_peak)
325
+
326
+ mean_size = np.mean(sizes)
327
+ mean_peak = np.mean(peaks)
328
+
329
+ for size in sizes:
330
+ assert mean_size * 1.1 > size
331
+ assert chunk_size * 3 > size
332
+ assert chunk_size * 2 < size
333
+
334
+ for peak in peaks:
335
+ assert mean_peak * 1.1 > peak
336
+ assert chunk_size * 7 > peak
337
+ assert chunk_size * 4 < peak
338
+
339
+ assert max(peaks) < large_ds.nbytes
340
+
341
+ tracemalloc.stop()