xarray-sql 0.2.0__cp310-cp310-manylinux_2_28_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +10 -0
- xarray_sql/_native.cpython-310-aarch64-linux-gnu.so +4 -0
- xarray_sql/core.py +48 -0
- xarray_sql/df.py +172 -0
- xarray_sql/df_test.py +341 -0
- xarray_sql/reader.py +253 -0
- xarray_sql/reader_test.py +944 -0
- xarray_sql/sql.py +18 -0
- xarray_sql/sql_test.py +194 -0
- xarray_sql-0.2.0.dist-info/METADATA +224 -0
- xarray_sql-0.2.0.dist-info/RECORD +14 -0
- xarray_sql-0.2.0.dist-info/WHEEL +4 -0
- xarray_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.2.0.dist-info/sboms/xarray_sql.cyclonedx.json +11420 -0
xarray_sql/__init__.py
ADDED
xarray_sql/core.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import typing as t
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import xarray as xr
|
|
6
|
+
|
|
7
|
+
Row = t.List[t.Any]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# deprecated
|
|
11
|
+
def get_columns(ds: xr.Dataset) -> t.List[str]:
|
|
12
|
+
return list(ds.sizes.keys()) + list(ds.data_vars.keys())
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Deprecated
|
|
16
|
+
def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
|
|
17
|
+
dim_keys, dim_vals = zip(*ds.sizes.items())
|
|
18
|
+
|
|
19
|
+
for idx in itertools.product(*(range(d) for d in dim_vals)):
|
|
20
|
+
coord_idx = dict(zip(dim_keys, idx))
|
|
21
|
+
data = ds.isel(coord_idx)
|
|
22
|
+
coord_data = [ds.coords[v][coord_idx[v]] for v in dim_keys]
|
|
23
|
+
row = [v.values for v in coord_data + list(data.data_vars.values())]
|
|
24
|
+
yield row
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Deprecated
|
|
28
|
+
def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
|
|
29
|
+
"""Unravel with unbounded memory (as a NumPy Array)."""
|
|
30
|
+
dim_keys, dim_vals = zip(*ds.sizes.items())
|
|
31
|
+
columns = get_columns(ds)
|
|
32
|
+
|
|
33
|
+
N = np.prod([d for d in dim_vals])
|
|
34
|
+
|
|
35
|
+
out = np.recarray((N,), dtype=[(c, ds[c].dtype) for c in columns])
|
|
36
|
+
|
|
37
|
+
for name, da in ds.items():
|
|
38
|
+
out[name] = da.values.ravel()
|
|
39
|
+
|
|
40
|
+
prod_vals = (ds.coords[k].values for k in dim_keys)
|
|
41
|
+
coords = np.array(np.meshgrid(*prod_vals), dtype=int).T.reshape(
|
|
42
|
+
-1, len(dim_keys)
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
for i, d in enumerate(dim_keys):
|
|
46
|
+
out[d] = coords[:, i]
|
|
47
|
+
|
|
48
|
+
return out
|
xarray_sql/df.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import typing as t
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import pyarrow as pa
|
|
8
|
+
import xarray as xr
|
|
9
|
+
from datafusion.context import ArrowStreamExportable
|
|
10
|
+
|
|
11
|
+
Block = t.Dict[t.Hashable, slice]
|
|
12
|
+
Chunks = t.Optional[t.Dict[str, int]]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Borrowed from Xarray
|
|
16
|
+
def _get_chunk_slicer(
|
|
17
|
+
dim: t.Hashable, chunk_index: t.Mapping, chunk_bounds: t.Mapping
|
|
18
|
+
):
|
|
19
|
+
if dim in chunk_index:
|
|
20
|
+
which_chunk = chunk_index[dim]
|
|
21
|
+
return slice(
|
|
22
|
+
chunk_bounds[dim][which_chunk], chunk_bounds[dim][which_chunk + 1]
|
|
23
|
+
)
|
|
24
|
+
return slice(None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Adapted from Xarray `map_blocks` implementation.
|
|
28
|
+
def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
|
|
29
|
+
"""Compute block slices for a chunked Dataset."""
|
|
30
|
+
if chunks is not None:
|
|
31
|
+
for_chunking = ds.copy(data=None, deep=False).chunk(chunks)
|
|
32
|
+
chunks = for_chunking.chunks
|
|
33
|
+
del for_chunking
|
|
34
|
+
else:
|
|
35
|
+
chunks = ds.chunks
|
|
36
|
+
|
|
37
|
+
assert chunks, "Dataset `ds` must be chunked or `chunks` must be provided."
|
|
38
|
+
|
|
39
|
+
# chunks is Dict[str, Tuple[int, ...]] from xarray
|
|
40
|
+
chunk_bounds = {
|
|
41
|
+
dim: np.cumsum((0,) + tuple(c)) # type: ignore[arg-type]
|
|
42
|
+
for dim, c in chunks.items()
|
|
43
|
+
}
|
|
44
|
+
ichunk = {dim: range(len(tuple(c))) for dim, c in chunks.items()} # type: ignore[arg-type]
|
|
45
|
+
ick, icv = zip(*ichunk.items()) # Makes same order of keys and val.
|
|
46
|
+
chunk_idxs = (dict(zip(ick, i)) for i in itertools.product(*icv))
|
|
47
|
+
blocks = (
|
|
48
|
+
{
|
|
49
|
+
dim: _get_chunk_slicer(dim, chunk_index, chunk_bounds)
|
|
50
|
+
for dim in ds.dims
|
|
51
|
+
}
|
|
52
|
+
for chunk_index in chunk_idxs
|
|
53
|
+
)
|
|
54
|
+
yield from blocks
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
|
|
58
|
+
"""Explodes a dataset into its chunks."""
|
|
59
|
+
yield from (ds.isel(b) for b in block_slices(ds, chunks=chunks))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _block_len(block: Block) -> int:
|
|
63
|
+
return int(np.prod([v.stop - v.start for v in block.values()]))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def from_map_batched(
|
|
67
|
+
func: t.Callable[..., pd.DataFrame],
|
|
68
|
+
*iterables,
|
|
69
|
+
args: t.Optional[t.Tuple] = None,
|
|
70
|
+
schema: pa.Schema = None,
|
|
71
|
+
**kwargs,
|
|
72
|
+
) -> pa.RecordBatchReader:
|
|
73
|
+
"""Create a PyArrow RecordBatchReader by mapping a function over iterables.
|
|
74
|
+
|
|
75
|
+
This is equivalent to dask's from_map but returns a PyArrow
|
|
76
|
+
RecordBatchReader that can be used with DataFusion. It iterates over
|
|
77
|
+
RecordBatches which are created via the `func` one-at-a-time.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
func: Function to apply to each element of the iterables. Currently, the function
|
|
81
|
+
must return a Pandas DataFrame.
|
|
82
|
+
*iterables: Iterable objects to map the function over.
|
|
83
|
+
schema: Optional schema needed for the RecordBatchReader.
|
|
84
|
+
args: Additional positional arguments to pass to func.
|
|
85
|
+
**kwargs: Additional keyword arguments to pass to func.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
A PyArrow RecordBatchReader containing the stream of RecordBatches.
|
|
89
|
+
"""
|
|
90
|
+
if args is None:
|
|
91
|
+
args = ()
|
|
92
|
+
|
|
93
|
+
def map_batches():
|
|
94
|
+
for items in zip(*iterables):
|
|
95
|
+
df = func(*items, *args, **kwargs)
|
|
96
|
+
yield pa.RecordBatch.from_pandas(df, schema=schema)
|
|
97
|
+
|
|
98
|
+
return pa.RecordBatchReader.from_batches(schema, map_batches())
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def from_map(
|
|
102
|
+
func: t.Callable, *iterables, args: t.Optional[t.Tuple] = None, **kwargs
|
|
103
|
+
) -> pa.Table:
|
|
104
|
+
"""Create a PyArrow Table by mapping a function over iterables.
|
|
105
|
+
|
|
106
|
+
This is equivalent to dask's from_map but returns a PyArrow Table
|
|
107
|
+
that can be used with DataFusion instead of a Dask DataFrame.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
func: Function to apply to each element of the iterables.
|
|
111
|
+
*iterables: Iterable objects to map the function over.
|
|
112
|
+
args: Additional positional arguments to pass to func.
|
|
113
|
+
**kwargs: Additional keyword arguments to pass to func.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
A PyArrow Table containing the concatenated results.
|
|
117
|
+
"""
|
|
118
|
+
if args is None:
|
|
119
|
+
args = ()
|
|
120
|
+
|
|
121
|
+
# Apply the function to each combination of iterable elements
|
|
122
|
+
results = []
|
|
123
|
+
for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
|
|
124
|
+
if isinstance(items, tuple):
|
|
125
|
+
result = func(*items, *args, **kwargs)
|
|
126
|
+
else:
|
|
127
|
+
result = func(items, *args, **kwargs)
|
|
128
|
+
|
|
129
|
+
# Convert result to PyArrow Table
|
|
130
|
+
if isinstance(result, pd.DataFrame):
|
|
131
|
+
pa_table = pa.Table.from_pandas(result)
|
|
132
|
+
elif isinstance(result, pa.Table):
|
|
133
|
+
pa_table = result
|
|
134
|
+
else:
|
|
135
|
+
# Try to convert to pandas first, then to PyArrow
|
|
136
|
+
try:
|
|
137
|
+
df = pd.DataFrame(result)
|
|
138
|
+
pa_table = pa.Table.from_pandas(df)
|
|
139
|
+
except Exception as e:
|
|
140
|
+
raise ValueError(
|
|
141
|
+
f"Cannot convert function result to PyArrow Table: {e}"
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
results.append(pa_table)
|
|
145
|
+
|
|
146
|
+
# Concatenate all results
|
|
147
|
+
if not results:
|
|
148
|
+
raise ValueError("No results to concatenate")
|
|
149
|
+
|
|
150
|
+
return pa.concat_tables(results)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def pivot(ds: xr.Dataset) -> pd.DataFrame:
|
|
154
|
+
"""Converts an xarray Dataset to a pandas DataFrame."""
|
|
155
|
+
return ds.to_dataframe().reset_index() # type: ignore[no-any-return]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _parse_schema(ds) -> pa.Schema:
|
|
159
|
+
"""Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
|
|
160
|
+
columns = []
|
|
161
|
+
|
|
162
|
+
for coord_name, coord_var in ds.coords.items():
|
|
163
|
+
# Only include dimension coordinates
|
|
164
|
+
if coord_name in ds.dims:
|
|
165
|
+
pa_type = pa.from_numpy_dtype(coord_var.dtype)
|
|
166
|
+
columns.append(pa.field(coord_name, pa_type))
|
|
167
|
+
|
|
168
|
+
for var_name, var in ds.data_vars.items():
|
|
169
|
+
pa_type = pa.from_numpy_dtype(var.dtype)
|
|
170
|
+
columns.append(pa.field(var_name, pa_type))
|
|
171
|
+
|
|
172
|
+
return pa.schema(columns)
|
xarray_sql/df_test.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import tracemalloc
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pyarrow as pa
|
|
7
|
+
import pytest
|
|
8
|
+
import xarray as xr
|
|
9
|
+
|
|
10
|
+
from .reader import read_xarray
|
|
11
|
+
from .df import explode, block_slices, from_map, pivot, from_map_batched
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def rand_wx(start: str, end: str) -> xr.Dataset:
|
|
15
|
+
np.random.seed(42)
|
|
16
|
+
lat = np.linspace(-90, 90, num=720)
|
|
17
|
+
lon = np.linspace(-180, 180, num=1440)
|
|
18
|
+
time = pd.date_range(start, end, freq="h")
|
|
19
|
+
level = np.array([1000, 500], dtype=np.int32)
|
|
20
|
+
reference_time = pd.Timestamp(start)
|
|
21
|
+
temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
|
|
22
|
+
precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
|
|
23
|
+
return xr.Dataset(
|
|
24
|
+
data_vars=dict(
|
|
25
|
+
temperature=(["lat", "lon", "time", "level"], temperature),
|
|
26
|
+
precipitation=(["lat", "lon", "time", "level"], precipitation),
|
|
27
|
+
),
|
|
28
|
+
coords=dict(
|
|
29
|
+
lat=lat,
|
|
30
|
+
lon=lon,
|
|
31
|
+
time=time,
|
|
32
|
+
level=level,
|
|
33
|
+
reference_time=reference_time,
|
|
34
|
+
),
|
|
35
|
+
attrs=dict(description="Random weather."),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def create_large_dataset(time_steps=1000, lat_points=100, lon_points=100):
|
|
40
|
+
"""Create a large xarray dataset for memory testing."""
|
|
41
|
+
np.random.seed(42)
|
|
42
|
+
|
|
43
|
+
time = pd.date_range("2020-01-01", periods=time_steps, freq="h")
|
|
44
|
+
lat = np.linspace(-90, 90, lat_points)
|
|
45
|
+
lon = np.linspace(-180, 180, lon_points)
|
|
46
|
+
|
|
47
|
+
temp_data = np.random.rand(time_steps, lat_points, lon_points) * 40 - 10
|
|
48
|
+
precip_data = np.random.rand(time_steps, lat_points, lon_points) * 100
|
|
49
|
+
|
|
50
|
+
return xr.Dataset(
|
|
51
|
+
{
|
|
52
|
+
"temperature": (["time", "lat", "lon"], temp_data),
|
|
53
|
+
"precipitation": (["time", "lat", "lon"], precip_data),
|
|
54
|
+
},
|
|
55
|
+
coords={"time": time, "lat": lat, "lon": lon},
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def adding_function(x, y):
|
|
60
|
+
"""Simple function that adds two values and returns a DataFrame."""
|
|
61
|
+
result = pd.DataFrame({"x": [x], "y": [y], "sum": [x + y]})
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@pytest.fixture
|
|
66
|
+
def air():
|
|
67
|
+
ds = xr.tutorial.open_dataset("air_temperature")
|
|
68
|
+
chunks = {"time": 240}
|
|
69
|
+
return ds.chunk(chunks)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.fixture
|
|
73
|
+
def air_small(air):
|
|
74
|
+
return air.isel(time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)).chunk(
|
|
75
|
+
{"time": 240}
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@pytest.fixture
|
|
80
|
+
def randwx():
|
|
81
|
+
return rand_wx("1995-01-13T00", "1995-01-13T01")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@pytest.fixture
|
|
85
|
+
def large_ds():
|
|
86
|
+
return create_large_dataset().chunk({"time": 25})
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_explode_cardinality(air):
|
|
90
|
+
dss = explode(air)
|
|
91
|
+
assert len(list(dss)) == np.prod([len(c) for c in air.chunks.values()])
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_explode_dim_sizes_one(air):
|
|
95
|
+
chunks = {"time": 240}
|
|
96
|
+
ds = next(iter(explode(air)))
|
|
97
|
+
for k, v in chunks.items():
|
|
98
|
+
assert k in ds.dims
|
|
99
|
+
assert v == ds.sizes[k]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@pytest.mark.skip(reason="TODO(alxmrs): Why is this test slow?")
|
|
103
|
+
def test_explode_dim_sizes_all(air):
|
|
104
|
+
dss = explode(air)
|
|
105
|
+
assert [tuple(ds.dims.values()) for ds in dss] == list(
|
|
106
|
+
itertools.product(*air.chunksizes.values())
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_explode_data_equal_one_first(air):
|
|
111
|
+
ds = next(iter(explode(air)))
|
|
112
|
+
iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
|
|
113
|
+
assert air.isel(iselection).equals(ds)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_explode_data_equal_one_last(air):
|
|
117
|
+
dss = list(explode(air))
|
|
118
|
+
ds = dss[-1]
|
|
119
|
+
|
|
120
|
+
# For the last chunk, we need to calculate where it actually starts
|
|
121
|
+
# The original logic slice(0, s) only works for the first chunk
|
|
122
|
+
iselection = {}
|
|
123
|
+
for dim in ds.dims:
|
|
124
|
+
# Get chunk boundaries
|
|
125
|
+
chunk_bounds = np.cumsum((0,) + air.chunks[dim])
|
|
126
|
+
# Last chunk index
|
|
127
|
+
last_chunk_idx = len(air.chunks[dim]) - 1
|
|
128
|
+
# Calculate actual start and end positions
|
|
129
|
+
start = chunk_bounds[last_chunk_idx]
|
|
130
|
+
end = chunk_bounds[last_chunk_idx + 1]
|
|
131
|
+
iselection[dim] = slice(start, end)
|
|
132
|
+
|
|
133
|
+
assert air.isel(iselection).equals(ds)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_from_map_basic():
|
|
137
|
+
def make_df(x):
|
|
138
|
+
return pd.DataFrame({"value": [x, x * 2], "index": [0, 1]})
|
|
139
|
+
|
|
140
|
+
result = from_map(make_df, [1, 2, 3])
|
|
141
|
+
assert isinstance(result, pa.Table)
|
|
142
|
+
assert len(result) == 6
|
|
143
|
+
assert result.column_names == ["value", "index"]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_from_map_multiple_iterables():
|
|
147
|
+
def add_values(x, y):
|
|
148
|
+
return pd.DataFrame({"sum": [x + y], "x": [x], "y": [y]})
|
|
149
|
+
|
|
150
|
+
result = from_map(add_values, [1, 2], [10, 20])
|
|
151
|
+
assert isinstance(result, pa.Table)
|
|
152
|
+
assert len(result) == 2
|
|
153
|
+
|
|
154
|
+
df = result.to_pandas()
|
|
155
|
+
assert list(df["sum"]) == [11, 22]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_from_map_with_args():
|
|
159
|
+
def multiply_and_add(x, multiplier, add_value):
|
|
160
|
+
return pd.DataFrame({"result": [x * multiplier + add_value]})
|
|
161
|
+
|
|
162
|
+
result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
|
|
163
|
+
assert isinstance(result, pa.Table)
|
|
164
|
+
assert len(result) == 3
|
|
165
|
+
|
|
166
|
+
df = result.to_pandas()
|
|
167
|
+
assert list(df["result"]) == [12, 14, 16]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_from_map_with_pyarrow_tables():
|
|
171
|
+
def make_arrow_table(x):
|
|
172
|
+
df = pd.DataFrame({"value": [x]})
|
|
173
|
+
return pa.Table.from_pandas(df)
|
|
174
|
+
|
|
175
|
+
result = from_map(make_arrow_table, [1, 2, 3])
|
|
176
|
+
assert isinstance(result, pa.Table)
|
|
177
|
+
assert len(result) == 3
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_from_map_batched_basic_functionality(air_small):
|
|
181
|
+
blocks = list(block_slices(air_small, chunks={"time": 4, "lat": 3, "lon": 4}))
|
|
182
|
+
|
|
183
|
+
first_block_df = pivot(air_small.isel(blocks[0]))
|
|
184
|
+
expected_schema = pa.Schema.from_pandas(first_block_df)
|
|
185
|
+
|
|
186
|
+
reader = from_map_batched(
|
|
187
|
+
pivot, [air_small.isel(block) for block in blocks], schema=expected_schema
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
assert isinstance(reader, pa.RecordBatchReader)
|
|
191
|
+
assert reader.schema == expected_schema
|
|
192
|
+
|
|
193
|
+
batches = list(reader)
|
|
194
|
+
assert len(batches) > 0
|
|
195
|
+
for batch in batches:
|
|
196
|
+
assert batch.schema == expected_schema
|
|
197
|
+
assert len(batch) > 0
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_from_map_batched_multiple_iterables():
|
|
201
|
+
x_values = [1, 2, 3, 4, 5]
|
|
202
|
+
y_values = [10, 20, 30, 40, 50]
|
|
203
|
+
|
|
204
|
+
expected_schema = pa.schema(
|
|
205
|
+
[("x", pa.int64()), ("y", pa.int64()), ("sum", pa.int64())]
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
reader = from_map_batched(
|
|
209
|
+
adding_function, x_values, y_values, schema=expected_schema
|
|
210
|
+
)
|
|
211
|
+
table = reader.read_all()
|
|
212
|
+
df = table.to_pandas()
|
|
213
|
+
|
|
214
|
+
expected_df = pd.DataFrame(
|
|
215
|
+
{
|
|
216
|
+
"x": x_values,
|
|
217
|
+
"y": y_values,
|
|
218
|
+
"sum": [x + y for x, y in zip(x_values, y_values)],
|
|
219
|
+
}
|
|
220
|
+
)
|
|
221
|
+
pd.testing.assert_frame_equal(df, expected_df)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_from_map_batched_with_args_and_kwargs():
|
|
225
|
+
def multiply_and_add(x, multiplier, offset=0):
|
|
226
|
+
return pd.DataFrame({"x": [x], "result": [x * multiplier + offset]})
|
|
227
|
+
|
|
228
|
+
values = [1, 2, 3]
|
|
229
|
+
expected_schema = pa.schema([("x", pa.int64()), ("result", pa.int64())])
|
|
230
|
+
|
|
231
|
+
reader = from_map_batched(
|
|
232
|
+
multiply_and_add, values, args=(2,), offset=5, schema=expected_schema
|
|
233
|
+
)
|
|
234
|
+
table = reader.read_all()
|
|
235
|
+
df = table.to_pandas()
|
|
236
|
+
|
|
237
|
+
expected_df = pd.DataFrame({"x": [1, 2, 3], "result": [7, 9, 11]})
|
|
238
|
+
pd.testing.assert_frame_equal(df, expected_df)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def test_from_map_batched_empty_iterables():
|
|
242
|
+
empty_schema = pa.schema([("value", pa.int64())])
|
|
243
|
+
|
|
244
|
+
reader = from_map_batched(
|
|
245
|
+
lambda x: pd.DataFrame({"value": [x]}), [], schema=empty_schema
|
|
246
|
+
)
|
|
247
|
+
batches = list(reader)
|
|
248
|
+
assert len(batches) == 0
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def test_from_map_batched_consistency_with_regular_map(air_small):
|
|
252
|
+
blocks = list(block_slices(air_small, chunks={"time": 4, "lat": 3}))
|
|
253
|
+
datasets = [air_small.isel(block) for block in blocks]
|
|
254
|
+
|
|
255
|
+
first_df = pivot(datasets[0])
|
|
256
|
+
schema = pa.Schema.from_pandas(first_df)
|
|
257
|
+
|
|
258
|
+
reader = from_map_batched(pivot, datasets, schema=schema)
|
|
259
|
+
batched_table = reader.read_all()
|
|
260
|
+
|
|
261
|
+
regular_dfs = [pivot(ds) for ds in datasets]
|
|
262
|
+
regular_table = pa.Table.from_pandas(
|
|
263
|
+
pd.concat(regular_dfs, ignore_index=True)
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
assert batched_table.schema == regular_table.schema
|
|
267
|
+
assert len(batched_table) == len(regular_table)
|
|
268
|
+
|
|
269
|
+
batched_df = (
|
|
270
|
+
batched_table.to_pandas()
|
|
271
|
+
.sort_values(["time", "lat", "lon"])
|
|
272
|
+
.reset_index(drop=True)
|
|
273
|
+
)
|
|
274
|
+
regular_df = (
|
|
275
|
+
regular_table.to_pandas()
|
|
276
|
+
.sort_values(["time", "lat", "lon"])
|
|
277
|
+
.reset_index(drop=True)
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
pd.testing.assert_frame_equal(batched_df, regular_df)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def test_from_map_batched_integration_with_datafusion_via_read_xarray():
|
|
284
|
+
air = xr.tutorial.open_dataset("air_temperature")
|
|
285
|
+
air_small = air.isel(time=slice(0, 50), lat=slice(0, 10), lon=slice(0, 15))
|
|
286
|
+
air_chunked = air_small.chunk({"time": 25, "lat": 5, "lon": 8})
|
|
287
|
+
|
|
288
|
+
arrow_stream = read_xarray(
|
|
289
|
+
air_chunked, chunks={"time": 25, "lat": 5, "lon": 8}
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
assert hasattr(arrow_stream, "schema")
|
|
293
|
+
assert hasattr(arrow_stream, "__iter__")
|
|
294
|
+
|
|
295
|
+
table = arrow_stream.read_all()
|
|
296
|
+
assert len(table) > 0
|
|
297
|
+
|
|
298
|
+
expected_columns = {"time", "lat", "lon", "air"}
|
|
299
|
+
actual_columns = set(table.column_names)
|
|
300
|
+
assert expected_columns.issubset(actual_columns)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def test_read_xarray_loads_one_chunk_at_a_time(large_ds):
|
|
304
|
+
tracemalloc.start()
|
|
305
|
+
iterable = read_xarray(large_ds)
|
|
306
|
+
first_size, first_peak = tracemalloc.get_traced_memory()
|
|
307
|
+
tracemalloc.reset_peak()
|
|
308
|
+
|
|
309
|
+
sizes, peaks = [], []
|
|
310
|
+
|
|
311
|
+
first_chunk = large_ds.isel(next(block_slices(large_ds)))
|
|
312
|
+
chunk_size = first_chunk.nbytes
|
|
313
|
+
|
|
314
|
+
# Creating the iterator should be inexpensive -- less than one chunk.
|
|
315
|
+
# We multiply by constant factors because chunks have additional overhead
|
|
316
|
+
assert first_size < chunk_size * 3
|
|
317
|
+
assert first_peak < chunk_size * 6
|
|
318
|
+
|
|
319
|
+
for it in iterable:
|
|
320
|
+
_ = it
|
|
321
|
+
cur_size, cur_peak = tracemalloc.get_traced_memory()
|
|
322
|
+
tracemalloc.reset_peak()
|
|
323
|
+
sizes.append(cur_size)
|
|
324
|
+
peaks.append(cur_peak)
|
|
325
|
+
|
|
326
|
+
mean_size = np.mean(sizes)
|
|
327
|
+
mean_peak = np.mean(peaks)
|
|
328
|
+
|
|
329
|
+
for size in sizes:
|
|
330
|
+
assert mean_size * 1.1 > size
|
|
331
|
+
assert chunk_size * 3 > size
|
|
332
|
+
assert chunk_size * 2 < size
|
|
333
|
+
|
|
334
|
+
for peak in peaks:
|
|
335
|
+
assert mean_peak * 1.1 > peak
|
|
336
|
+
assert chunk_size * 7 > peak
|
|
337
|
+
assert chunk_size * 4 < peak
|
|
338
|
+
|
|
339
|
+
assert max(peaks) < large_ds.nbytes
|
|
340
|
+
|
|
341
|
+
tracemalloc.stop()
|